diff --git a/.agents/skills/local-review-codex/SKILL.md b/.agents/skills/local-review-codex/SKILL.md index cc932f7a4b..ec1277fe3a 100644 --- a/.agents/skills/local-review-codex/SKILL.md +++ b/.agents/skills/local-review-codex/SKILL.md @@ -1,6 +1,6 @@ --- name: local-review-codex -description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy, model, and reasoning effort as the codex-pr-review GitHub action. +description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy and reasoning effort as the codex-pr-review GitHub action, on a newer model. --- # Local Codex Review (pre-push) @@ -11,17 +11,18 @@ before the PR exists. Use this before `git push` on a non-trivial change. **Correspondence with CI** — identical: - Policy: `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test coverage). -- Model: `gpt-5.6-sol`, `model_reasoning_effort="xhigh"`. +- Reasoning effort: `model_reasoning_effort="xhigh"`. - Output: markdown starting with `## Codex Review`, findings tagged P0 / P1 / P2 with file:line. **Differences from CI** — local-only: +- Model is `gpt-6-astra`; CI stays on `gpt-5.6-sol`. Not an oversight to reconcile: `gpt-6-astra` is confirmed on the ChatGPT auth `codex login` uses locally, while CI authenticates with `OPENAI_API_KEY` (`codex-pr-review.yml` prefers it over `CODEX_AUTH_JSON`) and that tier is unverified for the model. Move CI once API access is confirmed, or once CI switches to `CODEX_AUTH_JSON`. - Scope is the current branch vs `main` at the merge-base, **including uncommitted changes** (CI reviews a pushed PR diff). - Sandbox is `read-only` (CI uses `danger-full-access` on an ephemeral runner). Codex reads the diff and files but cannot modify your working tree. - Fresh context is inherent: `codex exec` is a separate cold process, so it does not anchor on the current chat session — the same reason `local-review` insists on a subagent. ## Prerequisites -- `codex` CLI **>= 0.144.1** installed and authed (`codex login` or `OPENAI_API_KEY`). Older CLIs reject `gpt-5.6-sol` with "requires a newer version of Codex". Upgrade with `npm install --global @openai/codex@0.144.1` (may need `sudo` for a global install). Keep this in sync with the pin in `.github/workflows/codex-pr-review.yml`. +- `codex` CLI **>= 0.153.4** installed and authed via `codex login` (an `OPENAI_API_KEY` in the environment takes priority and may not reach `gpt-6-astra` — see the model note above). Older CLIs reject the model with "requires a newer version of Codex"; `run.sh` checks the version up front. Upgrade with `npm install --global @openai/codex@0.153.4` (may need `sudo` for a global install). This matches the pin in `.github/workflows/codex-pr-review.yml` — the CLI version is the same on both sides, only the model differs. - `git fetch` the base ref if it's stale, so the merge-base is accurate. ## Run diff --git a/.agents/skills/local-review-codex/run.sh b/.agents/skills/local-review-codex/run.sh index d6491099c2..948d3820eb 100755 --- a/.agents/skills/local-review-codex/run.sh +++ b/.agents/skills/local-review-codex/run.sh @@ -1,21 +1,44 @@ #!/usr/bin/env bash # Local Codex review — mirrors the .github/workflows/codex-pr-review.yml CI job, # but scoped to this branch's unpushed work (committed + uncommitted) so you can -# review before pushing. Same policy (REVIEW.md), same model (gpt-5.6-sol) and -# reasoning effort (xhigh) as CI. Runs read-only: Codex cannot modify your tree. +# review before pushing. Same policy (REVIEW.md) and reasoning effort (xhigh) as CI. +# +# The model deliberately differs from CI: gpt-6-astra is confirmed available on the +# ChatGPT auth `codex login` uses here, but CI authenticates with OPENAI_API_KEY and +# that tier is unverified for it, so codex-pr-review.yml stays on gpt-5.6-sol. # # Usage: run.sh [BASE_REF] (BASE_REF defaults to "main") set -euo pipefail +MODEL="gpt-6-astra" +CODEX_MIN="0.153.4" + BASE_REF="${1:-main}" REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" if ! command -v codex >/dev/null 2>&1; then - echo "codex CLI not found. Install with: npm install --global @openai/codex@0.144.1" >&2 + echo "codex CLI not found. Install with: npm install --global @openai/codex@$CODEX_MIN" >&2 exit 1 fi +# Older CLIs reject the model with an error that never names the CLI version as the +# cause, so check it up front rather than letting the exec fail opaquely. The `|| true` +# keeps an unrecognised --version format from aborting under `set -e`: an unparseable +# version means "cannot tell", which must fall through to the exec, not kill the review. +CODEX_VER="$(codex --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" +if [ -n "$CODEX_VER" ] && [ "$(printf '%s\n%s\n' "$CODEX_MIN" "$CODEX_VER" | sort -V | head -1)" != "$CODEX_MIN" ]; then + echo "codex $CODEX_VER is too old for $MODEL (need >= $CODEX_MIN). Upgrade with: npm install --global @openai/codex@$CODEX_MIN" >&2 + exit 1 +fi + +# codex prefers OPENAI_API_KEY over the ChatGPT credentials `codex login` stores, and +# that tier is not confirmed for $MODEL — the resulting failure names the model, not the +# auth that selected it. +if [ -n "${OPENAI_API_KEY:-}" ]; then + echo "warning: OPENAI_API_KEY is set and takes priority over 'codex login' credentials; $MODEL may be unavailable on that tier." >&2 +fi + # Resolve the base to a concrete commit, preferring a local ref but falling back to # the remote-tracking ref — checkouts (CI, single-branch clones) often have only # origin/main, not a local main. @@ -80,7 +103,7 @@ EOF codex exec \ -C "$REPO_ROOT" \ - -m gpt-5.6-sol \ + -m "$MODEL" \ -c 'model_reasoning_effort="xhigh"' \ -s read-only \ -o "$OUT" \ diff --git a/.claude/settings.json b/.claude/settings.json index a464ca3719..37aaeeb0f5 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -55,22 +55,18 @@ "mcp__claude_ai_Gmail__list_drafts" ], "deny": [ - "Read(.env)", - "Read(.env.*)", - "Read(**/.env)", - "Read(**/.env.*)", - "Read(**/secrets/**)", - "Read(**/*.pem)", - "Read(**/*.key)", - "Read(**/credentials.json)", - "Read(**/.secret*)", - "Read(**/.secrets*)", - "Read(**/*.secret)", - "Read(**/*.secrets)", "Edit(.env)", "Edit(.env.*)", "Edit(**/.env)", - "Edit(**/.env.*)" + "Edit(**/.env.*)", + "Edit(**/secrets/**)", + "Edit(**/*.pem)", + "Edit(**/*.key)", + "Edit(**/credentials.json)", + "Edit(**/.secret*)", + "Edit(**/.secrets*)", + "Edit(**/*.secret)", + "Edit(**/*.secrets)" ], "ask": [ "Bash(rmdir:*)", diff --git a/.github/actions/sign-attest-image/action.yml b/.github/actions/sign-attest-image/action.yml new file mode 100644 index 0000000000..de0133dac0 --- /dev/null +++ b/.github/actions/sign-attest-image/action.yml @@ -0,0 +1,56 @@ +name: Sign image and attach provenance +description: > + Keyless-signs a pushed image digest with cosign (index and per-arch + manifests) and records SLSA provenance as a GitHub artifact attestation + pushed to the registry. SBOMs are not generated here: the build step embeds + them as BuildKit attestation manifests (depot `sbom: true`), which the index + signature then covers. The calling job must already be logged in to the + registry and must have id-token: write, attestations: write and + packages: write permissions (write-all covers all three). +inputs: + image: + description: "Fully-qualified image name without tag, e.g. ghcr.io/windmill-labs/windmill" + required: true + digest: + description: "Pushed manifest digest (sha256:...) from build-push-action" + required: true +runs: + using: composite + steps: + - name: Preflight + shell: bash + env: + DIGEST: ${{ inputs.digest }} + run: | + if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then + echo "::error::No OIDC token available; the calling job needs id-token: write" + exit 1 + fi + case "$DIGEST" in + sha256:*) ;; + *) + echo "::error::digest '$DIGEST' is not a sha256: digest" + exit 1 + ;; + esac + + # cosign v2 writes the classic sha256-.sig tag format that the + # installed base of cosign clients can verify; v3's bundle format cannot be + # verified by v2 clients yet, so stay on v2 until v3 verification is common. + - uses: sigstore/cosign-installer@v4.1.2 + with: + cosign-release: "v2.6.5" + + - name: Cosign keyless sign (index + per-arch manifests) + shell: bash + env: + IMAGE: ${{ inputs.image }} + DIGEST: ${{ inputs.digest }} + run: cosign sign --yes --recursive "${IMAGE}@${DIGEST}" + + - name: SLSA provenance (GitHub artifact attestation) + uses: actions/attest-build-provenance@v4 + with: + subject-name: ${{ inputs.image }} + subject-digest: ${{ inputs.digest }} + push-to-registry: true diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cdd0e2a212..9f65c2ae47 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -61,7 +61,7 @@ jobs: bun-version: 1.4.0 - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "24" - uses: astral-sh/setup-uv@v6.2.1 with: version: "0.11.24" diff --git a/.github/workflows/build_cli_image.yml b/.github/workflows/build_cli_image.yml index 5b71a786aa..de64a76583 100644 --- a/.github/workflows/build_cli_image.yml +++ b/.github/workflows/build_cli_image.yml @@ -13,9 +13,13 @@ permissions: contents: read id-token: write packages: write + attestations: write jobs: publish_cli: + # a tag-targeted dispatch would republish the release tags unsigned, + # un-verifying the release; to republish a release, re-push its tag + if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/') runs-on: ubicloud steps: - uses: actions/checkout@v4 @@ -42,14 +46,23 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: file: "./docker/DockerfileCli" platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ steps.meta.outputs.tags }} labels: | ${{ steps.meta.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 26b2d9aae8..7a3882df7c 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -219,7 +219,7 @@ jobs: - name: Install Codex CLI if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - run: npm install --global @openai/codex@0.144.1 + run: npm install --global @openai/codex@0.153.4 - name: Configure Codex auth if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 433f3a86b3..a8b20623bc 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -86,11 +86,13 @@ jobs: type=semver,pattern={{major}}.{{minor}} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} build-args: | features=ce WM_BUILD_VERSION=${{ github.sha }} @@ -100,6 +102,13 @@ jobs: labels: | ${{ steps.meta-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} + build_ee: runs-on: ubicloud if: (github.event_name != 'workflow_dispatch') || github.event.inputs.ee @@ -149,11 +158,13 @@ jobs: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} build-args: | features=ee WM_BUILD_VERSION=${{ github.sha }} @@ -164,6 +175,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee + digest: ${{ steps.docker_build.outputs.digest }} + attach_amd64_binary_to_release: needs: [build, build_ee] runs-on: ubicloud @@ -358,6 +376,21 @@ jobs: docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main + - uses: sigstore/cosign-installer@v4.1.2 + if: startsWith(github.ref, 'refs/tags/v') + with: + cosign-release: "v2.6.5" + # end-to-end release guard: the version tag pushed by this run must + # verify against this exact run's identity (the mutable :latest/:dev + # tags race with concurrent main builds, so they are not asserted here) + - name: Verify release image is signed + if: startsWith(github.ref, 'refs/tags/v') + run: | + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/windmill-labs/windmill/.github/workflows/docker-image.yml@${GITHUB_REF}" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF_NAME#v}" + tag_latest_ee: runs-on: ubicloud needs: [run_integration_test, build_ee] @@ -379,6 +412,21 @@ jobs: docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main + - uses: sigstore/cosign-installer@v4.1.2 + if: startsWith(github.ref, 'refs/tags/v') + with: + cosign-release: "v2.6.5" + # end-to-end release guard: the version tag pushed by this run must + # verify against this exact run's identity (the mutable :latest/:dev + # tags race with concurrent main builds, so they are not asserted here) + - name: Verify release ee image is signed + if: startsWith(github.ref, 'refs/tags/v') + run: | + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/windmill-labs/windmill/.github/workflows/docker-image.yml@${GITHUB_REF}" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${GITHUB_REF_NAME#v}" + verify_ee_image_vulnerabilities: runs-on: ubicloud needs: [tag_latest_ee] @@ -493,11 +541,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileCuda" tags: | ${{ steps.meta-ee-public.outputs.tags }} @@ -505,6 +555,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda + digest: ${{ steps.docker_build.outputs.digest }} + build_slim: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] @@ -537,17 +594,26 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlim" tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | ${{ steps.meta-ee-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-slim + digest: ${{ steps.docker_build.outputs.digest }} + build_ee_slim: needs: [build_ee] runs-on: ubicloud @@ -582,11 +648,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlimEe" tags: | ${{ steps.meta-ee-public.outputs.tags }} @@ -594,6 +662,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-slim + digest: ${{ steps.docker_build.outputs.digest }} + build_full: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] @@ -626,17 +701,26 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFull" tags: | ${{ steps.meta-public.outputs.tags }} labels: | ${{ steps.meta-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-full + digest: ${{ steps.docker_build.outputs.digest }} + build_ee_full: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] @@ -669,14 +753,23 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFullEe" tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/publish_extra.yml b/.github/workflows/publish_extra.yml index 49964c95d9..ca62933292 100644 --- a/.github/workflows/publish_extra.yml +++ b/.github/workflows/publish_extra.yml @@ -84,6 +84,9 @@ jobs: publish_extra: needs: [sleep, test_extra] + # a tag-targeted dispatch would republish the release tags unsigned, + # un-verifying the release; to republish a release, re-push its tag + if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/') runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 @@ -112,15 +115,24 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . file: ./docker/DockerfileExtra platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ steps.meta.outputs.tags }} labels: | ${{ steps.meta.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 249f8350ae..bbc3264f86 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.803.0" + ".": "1.811.1" } diff --git a/AGENTS.md b/AGENTS.md index 71c59b479d..f8c83ec465 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,8 +36,8 @@ Open-source platform for internal tools, workflows, API integrations, background - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. - **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead. -- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy, `gpt-5.6-sol`, `xhigh` reasoning; requires the `codex` CLI >= 0.144.1. -- **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc` +- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy and `xhigh` reasoning, on `gpt-6-astra` rather than the action's `gpt-5.6-sol`; requires the `codex` CLI >= 0.153.4. +- **Domain guides**: `.claude/skills/native-trigger/` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. diff --git a/CHANGELOG.md b/CHANGELOG.md index 33e24c0c00..0722ccb094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,199 @@ # Changelog +## [1.811.1](https://github.com/windmill-labs/windmill/compare/v1.811.0...v1.811.1) (2026-09-13) + + +### Bug Fixes + +* check kafka trigger topics against a set, not a one-pass iterator ([#11108](https://github.com/windmill-labs/windmill/issues/11108)) ([bf4fa2b](https://github.com/windmill-labs/windmill/commit/bf4fa2b174b8d4a5897b2aa0d108480442bd0e18)) +* let the hub_sync job read the uid and hub_base_url settings ([#11106](https://github.com/windmill-labs/windmill/issues/11106)) ([45102c8](https://github.com/windmill-labs/windmill/commit/45102c82659d86ca5ec6fd3aee57232f2348736c)) + +## [1.811.0](https://github.com/windmill-labs/windmill/compare/v1.810.0...v1.811.0) (2026-09-12) + + +### Features + +* make snowflake_oauth work as a dbt warehouse on every engine ([#11095](https://github.com/windmill-labs/windmill/issues/11095)) ([9fc50a2](https://github.com/windmill-labs/windmill/commit/9fc50a23fb75cb541c481247b7b25c206fb06d36)) + + +### Bug Fixes + +* accept any hub version of the git sync script in the token check ([#11099](https://github.com/windmill-labs/windmill/issues/11099)) ([670628b](https://github.com/windmill-labs/windmill/commit/670628b300ab119363adb5496ebfe3c6ccd80063)) +* bring back Publish to Hub for scripts ([#11097](https://github.com/windmill-labs/windmill/issues/11097)) ([864e5f0](https://github.com/windmill-labs/windmill/commit/864e5f02ec1c2dd16c74524f551f44485df10a20)) +* bundle deployed bun scripts whose only pin is on a dynamic import ([#11096](https://github.com/windmill-labs/windmill/issues/11096)) ([4afb9aa](https://github.com/windmill-labs/windmill/commit/4afb9aa677ac11d22e22e54bb6fd7881378b7005)) +* clear a stale git auto-pull failure and show the status time ([#11100](https://github.com/windmill-labs/windmill/issues/11100)) ([e877b5f](https://github.com/windmill-labs/windmill/commit/e877b5f2e81b1741aee90e843c8cccc22f9eef24)) +* stop a resource delete from taking variables it does not own ([#11102](https://github.com/windmill-labs/windmill/issues/11102)) ([2a21efa](https://github.com/windmill-labs/windmill/commit/2a21efa11b8307b331a8c20720028444dd3c62ff)) + +## [1.810.0](https://github.com/windmill-labs/windmill/compare/v1.809.0...v1.810.0) (2026-09-11) + + +### Features + +* **ai-sessions:** turn skills on by default, and group them by folder ([#11058](https://github.com/windmill-labs/windmill/issues/11058)) ([d8b9174](https://github.com/windmill-labs/windmill/commit/d8b9174235b97d0b4ef9f281e20e5704182d8dcb)) +* background and wait_seconds for run_script, skip preprocessor ([#11092](https://github.com/windmill-labs/windmill/issues/11092)) ([2939c2d](https://github.com/windmill-labs/windmill/commit/2939c2dd4b129640d87ef45c7a24c6f215a3239a)) +* give the chat the full MCP tool schema, and mark calls with the provider icon ([#11086](https://github.com/windmill-labs/windmill/issues/11086)) ([e7c6f85](https://github.com/windmill-labs/windmill/commit/e7c6f85553bd8efb0f7af0f488f615ac93c496ae)) +* let apps hide the viewer login status on public urls ([#11089](https://github.com/windmill-labs/windmill/issues/11089)) ([6056ec7](https://github.com/windmill-labs/windmill/commit/6056ec7148bce9f8ed171dd29f544696c335de7d)) +* remove the viewer login status badge from public apps ([#11090](https://github.com/windmill-labs/windmill/issues/11090)) ([75d7bee](https://github.com/windmill-labs/windmill/commit/75d7bee178886461fe49090d606a708f25c02d1a)) +* run a deployed flow through the chat's argument form ([#11085](https://github.com/windmill-labs/windmill/issues/11085)) ([b50de89](https://github.com/windmill-labs/windmill/commit/b50de8947908f1a5a4e9472afe6c0ecd25892e93)) +* run a flow test through the chat's argument form ([#11069](https://github.com/windmill-labs/windmill/issues/11069)) ([172d6c2](https://github.com/windmill-labs/windmill/commit/172d6c275b92b39d13848b543df9db10148e94ef)) + + +### Bug Fixes + +* attach TLS to gRPC OTLP exporters for https endpoints ([#11078](https://github.com/windmill-labs/windmill/issues/11078)) ([f915ed6](https://github.com/windmill-labs/windmill/commit/f915ed6a46e14341ddb213e869090edb68763032)) +* **dbt:** stop dbt sending anonymous usage stats from workers ([#11091](https://github.com/windmill-labs/windmill/issues/11091)) ([d539e86](https://github.com/windmill-labs/windmill/commit/d539e8674f2bf92d6c10b182ed4a1073714062c0)) +* keep pinned import versions of imported scripts in bun lockfiles ([#11082](https://github.com/windmill-labs/windmill/issues/11082)) ([57f8b08](https://github.com/windmill-labs/windmill/commit/57f8b0826ad61cb118d0cafbbc9203327d858940)) +* let admins and background sync reach private git hosts ([#11084](https://github.com/windmill-labs/windmill/issues/11084)) ([fa53099](https://github.com/windmill-labs/windmill/commit/fa53099e2b87a8676c5d6a18e77844e17ff45efd)) +* serve instance env settings at the documented /settings/local path ([#11075](https://github.com/windmill-labs/windmill/issues/11075)) ([f8f7c00](https://github.com/windmill-labs/windmill/commit/f8f7c0009f32c1440566420725b12e78ca804b03)) +* show symlinked files in the git repo viewer ([#11081](https://github.com/windmill-labs/windmill/issues/11081)) ([e6d4f44](https://github.com/windmill-labs/windmill/commit/e6d4f44a6122dab47fbee2a2a2b4330a62841bed)) +* support gzip and zstd compression for OTLP export over gRPC ([#11077](https://github.com/windmill-labs/windmill/issues/11077)) ([b156778](https://github.com/windmill-labs/windmill/commit/b156778da24e3827f723e503f80df68de87ddf7e)) +* unpin only the specifiers in the bundle a bun modules run executes ([#11083](https://github.com/windmill-labs/windmill/issues/11083)) ([30ffdbe](https://github.com/windmill-labs/windmill/commit/30ffdbecc15270562ceed3030c50a1cf81b1195c)) + + +### Performance Improvements + +* lazy-load the low-code runtime on public app pages ([#11087](https://github.com/windmill-labs/windmill/issues/11087)) ([e651b4c](https://github.com/windmill-labs/windmill/commit/e651b4cd63c3bd64a8739c9f60c6dfa437f19b4b)) + +## [1.809.0](https://github.com/windmill-labs/windmill/compare/v1.808.0...v1.809.0) (2026-09-10) + + +### Features + +* add a minimal skin for the approval page and slack/teams ([#11061](https://github.com/windmill-labs/windmill/issues/11061)) ([63cb46d](https://github.com/windmill-labs/windmill/commit/63cb46d7bb9db1d996aa38e06bd3afebc60111bd)) +* live queue status per tag and bounded queue metric charts ([#11067](https://github.com/windmill-labs/windmill/issues/11067)) ([569adb8](https://github.com/windmill-labs/windmill/commit/569adb85c1885d289e80a70a166f0f74e6d5ba83)) +* **otel:** read the OTLP metrics temporality preference ([#11064](https://github.com/windmill-labs/windmill/issues/11064)) ([2f88769](https://github.com/windmill-labs/windmill/commit/2f8876908719b3640d7cfc9364a7b0f7145fc356)) +* **otel:** support standard OTEL resource attribute env vars ([#10974](https://github.com/windmill-labs/windmill/issues/10974)) ([0a40eea](https://github.com/windmill-labs/windmill/commit/0a40eea37a7dbde5fc4760d6333d81186dfee255)) +* report script metadata with no content file in wmill lint ([#11053](https://github.com/windmill-labs/windmill/issues/11053)) ([8820b9f](https://github.com/windmill-labs/windmill/commit/8820b9fc644c6620c50517cd0d902015e2e670e2)) +* show the workspace an operator is in, and let them switch ([#11059](https://github.com/windmill-labs/windmill/issues/11059)) ([385086f](https://github.com/windmill-labs/windmill/commit/385086ffc21c72cd07624584932a4a301c23a732)) +* tuck other users' spaces into a collapsible home tree row ([#11073](https://github.com/windmill-labs/windmill/issues/11073)) ([d87f089](https://github.com/windmill-labs/windmill/commit/d87f089288996af9ea7e3b017a5ef35d4ded880d)) + + +### Bug Fixes + +* **ai-chat:** test_run_flow could test a different flow than the one asked ([#11066](https://github.com/windmill-labs/windmill/issues/11066)) ([fa73539](https://github.com/windmill-labs/windmill/commit/fa73539839071491fb2cbad9242f52ad22b975bf)) +* bound list_jobs runtime and paginate runs on the sorted column ([#11072](https://github.com/windmill-labs/windmill/issues/11072)) ([f517402](https://github.com/windmill-labs/windmill/commit/f51740253871960b55ab2aa8989e8df3fbde0351)) +* **frontend:** clear the flow graph selection through xyflow's store ([#11056](https://github.com/windmill-labs/windmill/issues/11056)) ([b4be8bc](https://github.com/windmill-labs/windmill/commit/b4be8bc5354fbd3a47c267c331aaf603c0f90e6e)) +* **frontend:** recompute dataflow edges when selecting a step ([#11070](https://github.com/windmill-labs/windmill/issues/11070)) ([08d876a](https://github.com/windmill-labs/windmill/commit/08d876aebf32ebb995a8c1839aa87794c0176bed)) +* **frontend:** restore heading sizes in note markdown and keep group notes on id change ([#11047](https://github.com/windmill-labs/windmill/issues/11047)) ([e63072c](https://github.com/windmill-labs/windmill/commit/e63072c216383700a23504be89782f4c69657174)) +* give every table a primary key so the db can be logically replicated ([#11036](https://github.com/windmill-labs/windmill/issues/11036)) ([e62bfdc](https://github.com/windmill-labs/windmill/commit/e62bfdcd8c6f1389601ccd2b5809c5eff0ff262e)) +* keep an app's deployed policy on wmill push ([#11049](https://github.com/windmill-labs/windmill/issues/11049)) ([0af7675](https://github.com/windmill-labs/windmill/commit/0af7675588300863883a99c7f14bc53cafec8a7e)) +* refuse cross-site GET requests that run Hub scripts ([#11054](https://github.com/windmill-labs/windmill/issues/11054)) ([ab9efc8](https://github.com/windmill-labs/windmill/commit/ab9efc897cc94d61a67263772806b15e9225cece)) +* skip the deploy PR when the git sync push committed nothing ([#11076](https://github.com/windmill-labs/windmill/issues/11076)) ([8ecbd33](https://github.com/windmill-labs/windmill/commit/8ecbd339eef7314a93d599fcea4377299d4c493d)) +* space the trailing AI settings cards ([#11044](https://github.com/windmill-labs/windmill/issues/11044)) ([5d7eed1](https://github.com/windmill-labs/windmill/commit/5d7eed1c02b0966289cc8cca00a15d76dad187a5)) +* stop uv pip compile emitting lockfile annotations ([#11042](https://github.com/windmill-labs/windmill/issues/11042)) ([8aa8b7e](https://github.com/windmill-labs/windmill/commit/8aa8b7ee6c23f859f169637c0bfd3f8d964509e9)) +* surface why a private or untrusted git host is unreachable ([#11068](https://github.com/windmill-labs/windmill/issues/11068)) ([c57b18e](https://github.com/windmill-labs/windmill/commit/c57b18e46fcdd319213fe0a537cb18de418ca688)) + + +### Performance Improvements + +* index the FK columns that cascade on workspace delete ([#11052](https://github.com/windmill-labs/windmill/issues/11052)) ([9a563f6](https://github.com/windmill-labs/windmill/commit/9a563f6d72da28fe09b785cd0683e9698df72bba)) +* only write queue metrics when a tag's backlog changes ([#11055](https://github.com/windmill-labs/windmill/issues/11055)) ([9d75929](https://github.com/windmill-labs/windmill/commit/9d75929247ea2ec39286971fcbebf95d886194f3)) + +## [1.808.0](https://github.com/windmill-labs/windmill/compare/v1.807.0...v1.808.0) (2026-09-09) + + +### Features + +* run and test scripts from the AI chat through an argument form ([#11001](https://github.com/windmill-labs/windmill/issues/11001)) ([a6abf2c](https://github.com/windmill-labs/windmill/commit/a6abf2c8a744e9ee6acf4830cdfbb84f2f95cb36)) + +## [1.807.0](https://github.com/windmill-labs/windmill/compare/v1.806.0...v1.807.0) (2026-09-09) + + +### Features + +* add a dismissible instance-wide announcement banner ([#11037](https://github.com/windmill-labs/windmill/issues/11037)) ([abf4c6c](https://github.com/windmill-labs/windmill/commit/abf4c6c2348014ea4401b9be62b5b15e5800e879)) +* batch chained DDL statements into a single migration ([#11038](https://github.com/windmill-labs/windmill/issues/11038)) ([656e609](https://github.com/windmill-labs/windmill/commit/656e609595833bc854f845d35cf43157e76d732f)) +* create the cloud workspace in onboarding, and teach the empty home ([#10959](https://github.com/windmill-labs/windmill/issues/10959)) ([fd35b47](https://github.com/windmill-labs/windmill/commit/fd35b4765843879cb2254f402c142fd7510f1916)) +* link from the public run view to the authenticated run page ([#11041](https://github.com/windmill-labs/windmill/issues/11041)) ([09b81a9](https://github.com/windmill-labs/windmill/commit/09b81a9294bed2795a1a7b688d0b02e0a61957ea)) +* make guest access unavailable on the shared cloud ([#11040](https://github.com/windmill-labs/windmill/issues/11040)) ([0b63e0a](https://github.com/windmill-labs/windmill/commit/0b63e0a6929088ff25def4fd6547cf61668251a5)) + + +### Bug Fixes + +* ignore comments and continuations in python lockfiles ([#11035](https://github.com/windmill-labs/windmill/issues/11035)) ([90c4e10](https://github.com/windmill-labs/windmill/commit/90c4e1020a2ff896977648dd68b572413cea7709)) +* refetch an unparseable hub script cache entry instead of panicking ([#11033](https://github.com/windmill-labs/windmill/issues/11033)) ([88c3ebd](https://github.com/windmill-labs/windmill/commit/88c3ebdfc1325ffbea4521d854e71d231a6409c4)) +* stop a new AI session adopting a legacy sidebar chat ([#11039](https://github.com/windmill-labs/windmill/issues/11039)) ([1076b63](https://github.com/windmill-labs/windmill/commit/1076b638d987ba99c5c27e5478ca580534b9d572)) + +## [1.806.0](https://github.com/windmill-labs/windmill/compare/v1.805.0...v1.806.0) (2026-09-08) + + +### Features + +* bring gitlab repositories to parity for git sync ([#10938](https://github.com/windmill-labs/windmill/issues/10938)) ([9444049](https://github.com/windmill-labs/windmill/commit/9444049d6013c77a5f25f01a736eb5cd741fb3e6)) +* draw a dbt column trace, across projects and the pipeline boundary ([#11014](https://github.com/windmill-labs/windmill/issues/11014)) ([33f9828](https://github.com/windmill-labs/windmill/commit/33f9828c3ed15fe63fccedc1550584f15c0490ab)) +* durable dbt state per environment, and `--defer` onto it ([#10975](https://github.com/windmill-labs/windmill/issues/10975)) ([621fac5](https://github.com/windmill-labs/windmill/commit/621fac55abcd1859e8c8c06e5f4412e61bb85d59)) +* ingest dbt column lineage and real column schemas from the engine's parquet index ([#10977](https://github.com/windmill-labs/windmill/issues/10977)) ([0139467](https://github.com/windmill-labs/windmill/commit/0139467b01b82e4b3d474ca3f205358fa607d19a)) +* let a worker group override the dependency cache object store ([#11019](https://github.com/windmill-labs/windmill/issues/11019)) ([de98adf](https://github.com/windmill-labs/windmill/commit/de98adf055835ab7c4d6305e7d5d3bdab915876b)) +* **nativets:** bound fetch on a peer that never answers ([#11026](https://github.com/windmill-labs/windmill/issues/11026)) ([785277e](https://github.com/windmill-labs/windmill/commit/785277e0bb2ea77b71a89dd4d389d439dfcf9e03)) +* recognize `// volume:` mounts in PHP scripts ([#11018](https://github.com/windmill-labs/windmill/issues/11018)) ([f081fb1](https://github.com/windmill-labs/windmill/commit/f081fb10705cadf99e99dfa786d1cc2ebf0447db)) +* report a WAC task failure the workflow body never awaited ([#11017](https://github.com/windmill-labs/windmill/issues/11017)) ([3e3a41d](https://github.com/windmill-labs/windmill/commit/3e3a41d418d4ee3fe060bd3acf3324f311d89c8a)) +* retry a workflow-as-code task from its task options ([#11013](https://github.com/windmill-labs/windmill/issues/11013)) ([d3f305d](https://github.com/windmill-labs/windmill/commit/d3f305db982b7c5dc49babf9bec8b62adcd2557d)) + + +### Bug Fixes + +* chain redeploys onto a retired path's version history ([#11029](https://github.com/windmill-labs/windmill/issues/11029)) ([0b37226](https://github.com/windmill-labs/windmill/commit/0b372260787edb7e9627ad4fb6637ad5f1024e0a)) +* make the native trigger disable/enable toggle actually save ([#11024](https://github.com/windmill-labs/windmill/issues/11024)) ([448fce9](https://github.com/windmill-labs/windmill/commit/448fce93f743d5b2ef2a2d4496eb2ec238594a0a)) +* offload php signature parsing from async workers ([#11027](https://github.com/windmill-labs/windmill/issues/11027)) ([2cb02e3](https://github.com/windmill-labs/windmill/commit/2cb02e3b3398db49f16377dd79dde2dd6fb5cc02)) +* reduce php parser stack use in debug workers ([#11025](https://github.com/windmill-labs/windmill/issues/11025)) ([2ae8509](https://github.com/windmill-labs/windmill/commit/2ae8509b14f112c9ef5b71321e8fd52596a16c10)) + + +### Performance Improvements + +* reduce shared worker debug polling frames ([#11028](https://github.com/windmill-labs/windmill/issues/11028)) ([946756a](https://github.com/windmill-labs/windmill/commit/946756ae83deb4e7a93111edddbd4d98c596d5a9)) + +## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) + + +### Features + +* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9)) +* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63)) +* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c)) +* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4)) +* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a)) +* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab)) + + +### Bug Fixes + +* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728)) +* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198)) +* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43)) +* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c)) +* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac)) +* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de)) +* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f)) +* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f)) +* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13)) + +## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05) + + +### Features + +* **ai-sessions:** replace the context panel with an assistant settings modal ([#10919](https://github.com/windmill-labs/windmill/issues/10919)) ([fda7b3f](https://github.com/windmill-labs/windmill/commit/fda7b3f086619e3716e5894c07be127104174f1d)) +* **frontend:** group the agent form and edit saved agents as drafts ([#10880](https://github.com/windmill-labs/windmill/issues/10880)) ([f037c73](https://github.com/windmill-labs/windmill/commit/f037c73d104fffe7bb2640a5b1f2a92154c85e06)) +* guest app execution mode, a role that takes no seat ([#10929](https://github.com/windmill-labs/windmill/issues/10929)) ([fce635d](https://github.com/windmill-labs/windmill/commit/fce635d3c4c8962f448140ceb55a00fb99012701)) +* guest JWT entry for embedded apps ([#10954](https://github.com/windmill-labs/windmill/issues/10954)) ([8aab503](https://github.com/windmill-labs/windmill/commit/8aab5034a68a4aafb264b0e86d000ef58f4a8511)) +* instrument sandbox isolation, data tables and in-flow script edits ([#10981](https://github.com/windmill-labs/windmill/issues/10981)) ([130a2f7](https://github.com/windmill-labs/windmill/commit/130a2f74083ba1bd308beeb86e2cbbaa41fd3345)) +* make S3 permission rules reorderable by drag and drop ([#10958](https://github.com/windmill-labs/windmill/issues/10958)) ([2257b05](https://github.com/windmill-labs/windmill/commit/2257b05b2857c7ae2b5ae0b4f9004e2d4e757925)) +* reconcile IdP instance groups from the SSO groups claim ([#10957](https://github.com/windmill-labs/windmill/issues/10957)) ([79426a1](https://github.com/windmill-labs/windmill/commit/79426a1a68a6b19e12af4633b8a79d07a103a106)) + + +### Bug Fixes + +* deploy a relocked script version only when its lock changed ([#10966](https://github.com/windmill-labs/windmill/issues/10966)) ([1113828](https://github.com/windmill-labs/windmill/commit/11138284acc4c1d8673e86823c7f74c9e1f419e6)) +* **frontend:** render ordered lists in markdown descriptions ([#10973](https://github.com/windmill-labs/windmill/issues/10973)) ([a0295b2](https://github.com/windmill-labs/windmill/commit/a0295b20c436fd3f2bd6a6d294ae3cee005391e8)) +* keep braces inside string tool arguments out of JSON depth count ([#10965](https://github.com/windmill-labs/windmill/issues/10965)) ([3e3d2a6](https://github.com/windmill-labs/windmill/commit/3e3d2a636334146014926841949372083e6e8516)) +* keep the instance user editor popover inside the viewport ([#10979](https://github.com/windmill-labs/windmill/issues/10979)) ([1901d31](https://github.com/windmill-labs/windmill/commit/1901d3193bfc6a9e29d0b7c5389fef44ff9d3687)) +* meter WAC compute per segment, not the whole sleep ([#10985](https://github.com/windmill-labs/windmill/issues/10985)) ([5428710](https://github.com/windmill-labs/windmill/commit/54287102b22dd17903cdd4b48c5828875e5b9be4)) +* name the extension to load when duckdb autoload hits the fence ([#10972](https://github.com/windmill-labs/windmill/issues/10972)) ([64b6798](https://github.com/windmill-labs/windmill/commit/64b679879936e2ddf4dbc2f90edbd56e3893bd83)) +* **oauth:** show the account chooser on an explicit Google/Microsoft login ([#10961](https://github.com/windmill-labs/windmill/issues/10961)) ([9f7908e](https://github.com/windmill-labs/windmill/commit/9f7908e2622647388768b574083cc48a6e1990f1)) +* patch critical CVEs in the worker image ([#10962](https://github.com/windmill-labs/windmill/issues/10962)) ([b100606](https://github.com/windmill-labs/windmill/commit/b100606da6a61f2dbcb24516363f43643bc917e3)) +* render the MCP OAuth consent page without a workspace ([#10988](https://github.com/windmill-labs/windmill/issues/10988)) ([ebfac29](https://github.com/windmill-labs/windmill/commit/ebfac29096f12c4da2df45d5d82db83d352f3426)) +* stand the WAC park down for a cancel that beat it to the row ([#10990](https://github.com/windmill-labs/windmill/issues/10990)) ([f977f5b](https://github.com/windmill-labs/windmill/commit/f977f5bf8b1ac70d3afbdc8ad6fcbe072cc51ebc)) + ## [1.803.0](https://github.com/windmill-labs/windmill/compare/v1.802.0...v1.803.0) (2026-09-03) diff --git a/Dockerfile b/Dockerfile index 8ecee96725..6a290c3553 100644 --- a/Dockerfile +++ b/Dockerfile @@ -141,9 +141,9 @@ FROM ${DEBIAN_IMAGE} ARG TARGETPLATFORM ARG POWERSHELL_VERSION=7.5.0 ARG KUBECTL_VERSION=1.36.2 -ARG HELM_VERSION=3.21.2 +ARG HELM_VERSION=3.21.4 # NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte -ARG GO_VERSION=1.26.0 +ARG GO_VERSION=1.26.8 ARG APP=/usr/src/app ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true @@ -250,8 +250,8 @@ RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_r RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode -RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ +RUN curl -sL https://deb.nodesource.com/setup_24.x | bash - +RUN apt-get -y update && apt-get install -y --no-install-recommends curl procps nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* # go build is slower the first time it is ran, so we prewarm it in the build @@ -299,7 +299,7 @@ RUN bun install -g windmill-cli \ RUN curl -fsSL https://claude.ai/install.sh | bash \ && cp /root/.local/share/claude/versions/* /usr/bin/claude -COPY --from=php:8.3.30-cli-trixie /usr/local/bin/php /usr/bin/php +COPY --from=php:8.3.33-cli-trixie /usr/local/bin/php /usr/bin/php COPY --from=composer:2.9.5 /usr/bin/composer /usr/bin/composer # add the docker client to call docker from a worker if enabled diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 4501f24810..2e29b9064d 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -132,6 +132,10 @@ export async function runEval( setToolStatus: () => {}, removeToolStatus: () => {}, isPlanModeActive, + // Accepts the run form exactly as the model prefilled it: there is nobody here to + // edit the arguments, so a case can assert what the model proposed but never how + // it reacts to the user changing something. + requestRunArgs: async (_toolId, form) => form.args, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 338ed8504c..e5b275b86e 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -86,6 +86,7 @@ vi.mock('$lib/gen', async () => { previewBenchmarkSchedule, runBenchmarkDatatableSql, runBenchmarkFlowByPath, + runBenchmarkScriptByPath, runBenchmarkScriptPreview, updateBenchmarkDraft, listBenchmarkMcpTools @@ -279,6 +280,18 @@ vi.mock('$lib/gen', async () => { } return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody }) }, + runScriptByPath: async (data: { + workspace: string + path: string + requestBody?: Record + }) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkScriptByPath({ + workspace: data.workspace, + path: data.path, + args: data.requestBody + }) + : actual.JobService.runScriptByPath(data), runFlowByPath: async (data: { workspace: string path: string diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 94ecb5c029..71693156a9 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1974,6 +1974,144 @@ judgeChecklist: - deletes the deployed script via delete_workspace_item rather than a raw API endpoint +- id: global-test33-run-deployed-script-with-form + prompt: |- + Run the deployed script `f/evals/global/format_greeting` for me with the name "ada". + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_script + - call_api_endpoint + - write_script + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_script + field: args.name + stringIncludesAnyOf: + - ada + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed script through run_script rather than a preview test run or a raw API endpoint + - passes the name "ada" so the confirmation form comes up prefilled + +- id: global-test34-run-with-secret-from-variable + prompt: |- + Run the deployed `f/evals/global/billing_sync` for the account `acme` — use the billing + API token we already keep in the workspace. + initial: ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json + runtime: + maxTurns: 10 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + # A secret argument is filled by naming the variable that holds it: the value stays in + # the variable and only its path travels. A literal reaches the job as a reference too, + # minted on the way in, but it stays in the tool call the model emitted. + toolCallArgs: + - tool: run_script + field: args.api_token + stringIncludesAnyOf: + - "$var:f/evals/global/stripe_api_token" + - tool: run_script + field: args.account + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - fills the secret argument with a reference to the existing workspace variable rather than a literal token + - passes the account "acme" + - does not invent or guess the token's value + +- id: global-test35-run-deployed-flow-with-form + prompt: |- + Run the deployed flow `f/evals/global/notify_customer` for me — the customer is `acme`. + initial: ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_flow + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_flow + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_flow + - call_api_endpoint + - write_flow + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_flow + field: args.customer + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed flow through run_flow rather than a preview test run or a raw API endpoint + - passes the customer "acme" so the confirmation form comes up prefilled + +- id: global-test36-draft-flow-test-run-not-deployed + prompt: |- + Update the `calculate_total` step of `f/evals/global/process_invoice` so it applies 8% tax and + returns `subtotal`, `tax` and `total`, then run it to check it works. + Keep it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + path: f/evals/global/process_invoice + toolExpect: + # A one-step flow is as well checked by running the step as the whole flow, so both + # count: what matters is that the run is against the draft. + requiredToolsAnyOf: + - [test_run_flow, test_run_step] + # The draft is what the user asked to check, and run_flow would run the deployed + # version instead — the edit would not be in what ran. + forbiddenToolsUsed: + - run_flow + - call_api_endpoint + - deploy_workspace_item + # The judge cannot observe runs, and the edit's content is already pinned by + # global-test5 on this fixture; what this case guards is where the run went. + skipJudge: true + judgeChecklist: + - creates an AI draft of f/evals/global/process_invoice applying 8% tax + - does not deploy or save the draft + - id: global-undo-created-draft prompt: |- Create a draft Postgres resource at `u/admin/scratch_db` for host db.example.com port 5432, database `orders`, user `app`, and tell me what fields it ended up with. diff --git a/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json new file mode 100644 index 0000000000..ef166b0f9c --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "variables": [ + { + "path": "f/evals/global/stripe_api_token", + "value": "sk_live_do_not_leak_me", + "is_secret": true, + "description": "Token used by the billing sync job", + "labels": ["billing"] + } + ], + "scripts": [ + { + "path": "f/evals/global/billing_sync", + "summary": "Sync billing records", + "description": "Syncs billing records for one account, authenticating with an API token.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "account": { + "type": "string" + }, + "api_token": { + "type": "string", + "password": true, + "description": "API token to authenticate with" + } + }, + "required": ["account", "api_token"] + }, + "content": "export async function main(account: string, api_token: string) {\n return `synced ${account}`\n}\n" + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json new file mode 100644 index 0000000000..d8cbf37ace --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json @@ -0,0 +1,40 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/notify_customer", + "summary": "Notify a customer", + "description": "Sends a notification to the named customer.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "customer": { + "type": "string" + } + }, + "required": ["customer"] + }, + "value": { + "modules": [ + { + "id": "notify", + "summary": "Send the notification", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(customer: string) {\n return `Notified ${customer}`\n}\n", + "input_transforms": { + "customer": { + "type": "javascript", + "expr": "flow_input.customer" + } + } + } + } + ] + } + } + ] + } +} diff --git a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json b/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json deleted file mode 100644 index 9c1f8d00a7..0000000000 --- a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((\n SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM v2_job_queue\n WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1\n )))", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f" -} diff --git a/backend/.sqlx/query-0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed.json b/backend/.sqlx/query-0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed.json new file mode 100644 index 0000000000..67f2bbbe81 --- /dev/null +++ b/backend/.sqlx/query-0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COALESCE(dt.value->'database'->>'resource_type', 'unknown') AS \"kind!\",\n COUNT(*)::BIGINT AS \"count!\"\n FROM workspace_settings ws,\n LATERAL jsonb_each(ws.datatable->'datatables') dt\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n GROUP BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed" +} diff --git a/backend/.sqlx/query-06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac.json b/backend/.sqlx/query-06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac.json new file mode 100644 index 0000000000..843207dad7 --- /dev/null +++ b/backend/.sqlx/query-06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n (SELECT MIN(day) FROM guest_activity) AS since,\n (SELECT COUNT(DISTINCT email) FROM guest_activity\n WHERE day > CURRENT_DATE - 30)::INT AS \"guest_count!\",\n (SELECT COUNT(DISTINCT email) FROM guest_activity\n WHERE jwt_entry AND day > CURRENT_DATE - 30)::INT AS \"guest_jwt_count!\",\n (SELECT COUNT(DISTINCT workspace_id) FROM guest_activity\n WHERE day > CURRENT_DATE - 30)::INT AS \"guest_workspace_count!\",\n (SELECT COUNT(*) FROM workspace_settings ws JOIN workspace w ON w.id = ws.workspace_id\n WHERE ws.guest_access_enabled AND NOT w.deleted)::INT AS \"guest_enabled_workspace_count!\",\n (SELECT COUNT(*) FROM workspace WHERE NOT deleted)::INT AS \"workspace_count!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "since", + "type_info": "Date" + }, + { + "ordinal": 1, + "name": "guest_count!", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "guest_jwt_count!", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "guest_workspace_count!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "guest_enabled_workspace_count!", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "workspace_count!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null, + null, + null + ] + }, + "hash": "06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac" +} diff --git a/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json new file mode 100644 index 0000000000..db9d674fcf --- /dev/null +++ b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock',\n 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily',\n 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d" +} diff --git a/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json b/backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json similarity index 83% rename from backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json rename to backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json index 1c0035f7cb..8aecbed333 100644 --- a/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json +++ b/backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary,\n enabled\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -100,8 +105,9 @@ true, false, false, - true + true, + false ] }, - "hash": "6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa" + "hash": "0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd" } diff --git a/backend/.sqlx/query-0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801.json b/backend/.sqlx/query-0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801.json new file mode 100644 index 0000000000..359a7eeefd --- /dev/null +++ b/backend/.sqlx/query-0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = 0, suspend_until = NULL,\n started_at = coalesce(started_at, $2, now())\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801" +} diff --git a/backend/.sqlx/query-0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c.json b/backend/.sqlx/query-0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c.json new file mode 100644 index 0000000000..abdea8c942 --- /dev/null +++ b/backend/.sqlx/query-0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO guest_activity (email, workspace_id, day, jwt_entry)\n VALUES ($1, $2, CURRENT_DATE, true)\n ON CONFLICT (email, workspace_id, day)\n DO UPDATE SET jwt_entry = true, last_seen_at = now()\n WHERE NOT guest_activity.jwt_entry\n RETURNING 1 AS \"audited!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "audited!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c" +} diff --git a/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json new file mode 100644 index 0000000000..00b839de6a --- /dev/null +++ b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf" +} diff --git a/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json deleted file mode 100644 index 3f39982319..0000000000 --- a/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6" -} diff --git a/backend/.sqlx/query-1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1.json b/backend/.sqlx/query-1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1.json new file mode 100644 index 0000000000..3e4b85a916 --- /dev/null +++ b/backend/.sqlx/query-1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token\n (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id)\n VALUES ($1, $2, $3, $4, $5, now() + ($6 || ' seconds')::interval, false, $7, $8)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1" +} diff --git a/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json b/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json new file mode 100644 index 0000000000..f5bb8e14d1 --- /dev/null +++ b/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO git_sync_ci_test_check\n (workspace_id, poster_workspace_id, head_sha, head_ref, repo_url,\n repo_resource_path, check_run_id,\n created_at, concluded, conclusion, concluded_at, github_posted)\n VALUES ($1, $2, $3, $4, $5, $6, NULL, now(), false, NULL, NULL, false)\n ON CONFLICT (workspace_id, repo_resource_path, head_sha) DO UPDATE SET\n poster_workspace_id = EXCLUDED.poster_workspace_id,\n head_ref = EXCLUDED.head_ref,\n repo_url = EXCLUDED.repo_url,\n check_run_id = NULL,\n created_at = now(),\n concluded = false,\n conclusion = NULL,\n concluded_at = NULL,\n github_posted = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6" +} diff --git a/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json new file mode 100644 index 0000000000..7bb2cac5db --- /dev/null +++ b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6" +} diff --git a/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json b/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json new file mode 100644 index 0000000000..8647442530 --- /dev/null +++ b/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT test_script_path, tested_item_path, tested_item_kind, has_wildcard AS \"has_wildcard!\" FROM ci_test_reference WHERE workspace_id = $1 ORDER BY test_script_path, tested_item_kind, tested_item_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "test_script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "tested_item_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "tested_item_kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "has_wildcard!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9" +} diff --git a/backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json b/backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json new file mode 100644 index 0000000000..a78e343072 --- /dev/null +++ b/backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT u.username, u.email FROM workspace w JOIN usr u ON u.workspace_id = w.id AND u.email = w.owner WHERE w.id = $1 AND NOT u.disabled", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7" +} diff --git a/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json b/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json new file mode 100644 index 0000000000..b9cbc6c382 --- /dev/null +++ b/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d" +} diff --git a/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json new file mode 100644 index 0000000000..b60dd43fdf --- /dev/null +++ b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6" +} diff --git a/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json new file mode 100644 index 0000000000..77b00d9daa --- /dev/null +++ b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704" +} diff --git a/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json new file mode 100644 index 0000000000..6b4324af4c --- /dev/null +++ b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3" +} diff --git a/backend/.sqlx/query-209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23.json b/backend/.sqlx/query-209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23.json new file mode 100644 index 0000000000..56128dab99 --- /dev/null +++ b/backend/.sqlx/query-209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' = 'true')::BIGINT AS \"enabled!\",\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' = 'false')::BIGINT AS \"disabled!\",\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' IS NULL)::BIGINT AS \"unset!\"\n FROM workspace_settings ws,\n LATERAL jsonb_each(ws.datatable->'datatables') dt\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "disabled!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "unset!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23" +} diff --git a/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json new file mode 100644 index 0000000000..bdb54cfbb9 --- /dev/null +++ b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c" +} diff --git a/backend/.sqlx/query-2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520.json b/backend/.sqlx/query-2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520.json new file mode 100644 index 0000000000..c1ff273bcb --- /dev/null +++ b/backend/.sqlx/query-2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*)::BIGINT AS \"total!\",\n COUNT(DISTINCT (workspace_id, datatable))::BIGINT AS \"datatables!\"\n FROM datatable_migrations", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "datatables!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520" +} diff --git a/backend/.sqlx/query-21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe.json b/backend/.sqlx/query-21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe.json new file mode 100644 index 0000000000..ca9f2cbc29 --- /dev/null +++ b/backend/.sqlx/query-21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO guest_activity (email, workspace_id, day)\n VALUES ($1, $2, CURRENT_DATE)\n ON CONFLICT (email, workspace_id, day)\n DO UPDATE SET last_seen_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe" +} diff --git a/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json new file mode 100644 index 0000000000..50755132fc --- /dev/null +++ b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4,\n 'source', $4, $5, '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6,\n 'model', $6, $7, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c" +} diff --git a/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json new file mode 100644 index 0000000000..5f3d8300eb --- /dev/null +++ b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH slots AS (\n SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak,\n (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value,\n (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing,\n COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp,\n max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb\n FROM (\n SELECT id, t,\n CASE jsonb_typeof(value)\n WHEN 'number' THEN value::double precision\n WHEN 'object' THEN t - (value->>'since')::double precision\n END AS v,\n (value->>'since')::double precision AS since,\n (jsonb_typeof(value) = 'object')::int::double precision AS climbing,\n greatest(floor((t - $1::double precision) / $2::double precision), -1)::int\n AS slot\n FROM (\n SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > to_timestamp($1::double precision - $3::double precision)\n ) m\n ) s\n WHERE v IS NOT NULL\n GROUP BY id, slot\n )\n SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\",\n greatest(peak, CASE WHEN last_climb[1] < last THEN (\n SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision\n FROM metrics n\n WHERE n.id = slots.id AND n.id LIKE 'queue_%'\n AND n.created_at > to_timestamp(last_climb[1] + 0.001)\n AND n.created_at <= to_timestamp(last + 0.001)\n ) - last_climb[2] END) AS \"peak!\",\n first_value AS \"first_value!\", last_value AS \"last_value!\",\n last_climbing AS \"last_climbing!\", ramp AS \"ramp!\"\n FROM slots\n ORDER BY id, slot", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slot!", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "first!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "last!", + "type_info": "Float8" + }, + { + "ordinal": 4, + "name": "peak!", + "type_info": "Float8" + }, + { + "ordinal": 5, + "name": "first_value!", + "type_info": "Float8" + }, + { + "ordinal": 6, + "name": "last_value!", + "type_info": "Float8" + }, + { + "ordinal": 7, + "name": "last_climbing!", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "ramp!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Float8", + "Float8", + "Float8", + "Float8" + ] + }, + "nullable": [ + false, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863" +} diff --git a/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json new file mode 100644 index 0000000000..7148252002 --- /dev/null +++ b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188" +} diff --git a/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json b/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json new file mode 100644 index 0000000000..99c8262860 --- /dev/null +++ b/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n route_path = $1,\n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n allowed_origins = $6,\n authentication_resource_path = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n mode = $11,\n http_method = $12,\n static_asset_config = $13,\n edited_by = $14,\n permissioned_as = $15,\n request_type = $16,\n authentication_method = $17,\n summary = $18,\n description = $19,\n edited_at = now(),\n is_static_website = $20,\n error_handler_path = $21,\n error_handler_args = $22,\n retry = $23\n WHERE\n workspace_id = $24 AND\n path = $25\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908" +} diff --git a/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json new file mode 100644 index 0000000000..0937a312d1 --- /dev/null +++ b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094" +} diff --git a/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json b/backend/.sqlx/query-31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1.json similarity index 54% rename from backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json rename to backend/.sqlx/query-31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1.json index 774b47f825..556bd7c317 100644 --- a/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json +++ b/backend/.sqlx/query-31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n ))\n RETURNING token_prefix", + "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n ))\n RETURNING token_prefix", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5" + "hash": "31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1" } diff --git a/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json b/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json new file mode 100644 index 0000000000..941dc122d0 --- /dev/null +++ b/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f" +} diff --git a/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json new file mode 100644 index 0000000000..fbfab14017 --- /dev/null +++ b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db" +} diff --git a/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json b/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json new file mode 100644 index 0000000000..9f9cf036fb --- /dev/null +++ b/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r->>'script_path' as \"script_path\"\n FROM workspace_settings ws,\n jsonb_array_elements(\n CASE WHEN jsonb_typeof(ws.git_sync->'repositories') = 'array'\n THEN ws.git_sync->'repositories' END\n ) r\n WHERE ws.workspace_id = $1\n AND r->>'git_repo_resource_path' IN ($2, '$res:' || $2)\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1" +} diff --git a/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json b/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json new file mode 100644 index 0000000000..ebaaf0b755 --- /dev/null +++ b/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of_email' = $2 AND (value->'policy'->>'on_behalf_of' IS NULL OR value->'policy'->>'on_behalf_of' NOT LIKE 'g/%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da" +} diff --git a/backend/.sqlx/query-391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1.json b/backend/.sqlx/query-391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1.json new file mode 100644 index 0000000000..8fcb8140f4 --- /dev/null +++ b/backend/.sqlx/query-391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM app WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1" +} diff --git a/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json new file mode 100644 index 0000000000..28b34afa97 --- /dev/null +++ b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind,\n ingested_at)\n SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column,\n child_unique_id, child_column, lineage_kind, ingested_at\n FROM dbt_column_edge\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4" +} diff --git a/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json new file mode 100644 index 0000000000..843484dd86 --- /dev/null +++ b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "now!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753" +} diff --git a/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json new file mode 100644 index 0000000000..1268656e34 --- /dev/null +++ b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest, manifest_key, run_results,\n run_results_key, updated_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET\n job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest,\n manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results,\n run_results_key = EXCLUDED.run_results_key, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Uuid", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3" +} diff --git a/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json b/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json new file mode 100644 index 0000000000..2d0bb40c18 --- /dev/null +++ b/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET concluded = true, github_posted = true, concluded_at = now(),\n conclusion = COALESCE(conclusion, 'failure')\n WHERE (check_run_id IS NULL AND NOT concluded\n AND created_at < now() - make_interval(secs => $1))\n OR (concluded AND NOT github_posted\n AND concluded_at < now() - make_interval(secs => $2))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586" +} diff --git a/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json new file mode 100644 index 0000000000..7f950346ee --- /dev/null +++ b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4" +} diff --git a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json b/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json deleted file mode 100644 index 9f97b97a88..0000000000 --- a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH queue_metrics as (\n SELECT id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "values!", - "type_info": "JsonArray" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - null - ] - }, - "hash": "44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa" -} diff --git a/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json b/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json new file mode 100644 index 0000000000..526f3f5537 --- /dev/null +++ b/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59" +} diff --git a/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json b/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json new file mode 100644 index 0000000000..6bec80ccb2 --- /dev/null +++ b/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json @@ -0,0 +1,84 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n wrap_body = $1,\n raw_string = $2,\n allowed_origins = $3,\n authentication_resource_path = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n mode = $8,\n http_method = $9,\n static_asset_config = $10,\n edited_by = $11,\n permissioned_as = $12,\n request_type = $13,\n authentication_method = $14,\n summary = $15,\n description = $16,\n edited_at = now(),\n is_static_website = $17,\n error_handler_path = $18,\n error_handler_args = $19,\n retry = $20\n WHERE\n workspace_id = $21 AND\n path = $22\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc" +} diff --git a/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json new file mode 100644 index 0000000000..b2cfb1cd89 --- /dev/null +++ b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem->'credential'\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') AS elem\n WHERE workspace_id = $1 AND elem->>'git_repo_resource_path' IN ($2, $3)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6" +} diff --git a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json b/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json deleted file mode 100644 index 8d86ff3db6..0000000000 --- a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Text", - "Varchar", - "Int8", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57" -} diff --git a/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json new file mode 100644 index 0000000000..24e563d9ad --- /dev/null +++ b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "edge!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e" +} diff --git a/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json new file mode 100644 index 0000000000..913c16c31d --- /dev/null +++ b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text\n WHERE EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false\n AND language = 'dbt'\n AND (hash = $4 OR $4 = ANY(parent_hashes)))\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0" +} diff --git a/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json b/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json new file mode 100644 index 0000000000..8c784b664e --- /dev/null +++ b/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = now()\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4\n AND (tests_dispatched_at IS NULL\n OR (ci_test_job_ids IS NULL\n AND tests_dispatched_at < now() - make_interval(secs => $5)))\n RETURNING true as \"claimed!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "claimed!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da" +} diff --git a/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json b/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json new file mode 100644 index 0000000000..c8b8b94da4 --- /dev/null +++ b/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (\n SELECT h.sha FROM git_sync_synced_head h\n WHERE h.workspace_id = $1 AND h.repo_resource_path = $4 AND h.branch = $3\n ORDER BY h.synced_at DESC LIMIT 1\n ) = $2 AND NOT EXISTS (\n SELECT 1\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.kind = 'deploymentcallback'\n AND j.args->'__git_sync_auto_pull'->>'branch' = $3\n AND j.args->'__git_sync_auto_pull'->>'repo_resource_path'\n IN ($4, '$res:' || $4)\n ) AND NOT EXISTS (\n SELECT 1\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.kind IN ('dependencies', 'flowdependencies', 'appdependencies')\n ) as \"ready\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ready", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81" +} diff --git a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json similarity index 70% rename from backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json rename to backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json index da77dc1de5..134e58ed58 100644 --- a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json +++ b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", + "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.column_schema, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", "describe": { "columns": [ { @@ -80,21 +80,26 @@ }, { "ordinal": 15, - "name": "freshness", + "name": "column_schema", "type_info": "Jsonb" }, { "ordinal": 16, + "name": "freshness", + "type_info": "Jsonb" + }, + { + "ordinal": 17, "name": "raw_code", "type_info": "Text" }, { - "ordinal": 17, + "ordinal": 18, "name": "original_file_path", "type_info": "Text" }, { - "ordinal": 18, + "ordinal": 19, "name": "script_visible!", "type_info": "Bool" } @@ -127,8 +132,9 @@ true, true, true, + true, null ] }, - "hash": "9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12" + "hash": "4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5" } diff --git a/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json new file mode 100644 index 0000000000..21759e2fe5 --- /dev/null +++ b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT p.asset_path AS \"from_path!\", e.parent_column AS \"from_column!\",\n c.asset_path AS \"to_path!\", e.child_column AS \"to_column!\",\n e.lineage_kind AS \"kind!\"\n FROM unnest($2::text[], $3::bigint[], $4::uuid[])\n AS o(script_path, script_hash, job_id)\n JOIN dbt_column_edge e ON e.workspace_id = $1\n AND e.script_path = o.script_path\n AND e.job_id = o.job_id\n -- `=` still, with the NULL-to-NULL case\n -- spelled out and gated on the pin: a\n -- version-less row's hash is NULL on both\n -- sides, which `=` never matches, but\n -- `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound everywhere else.\n AND (e.script_hash = o.script_hash\n OR ($5::text IS NOT NULL\n AND o.script_hash IS NULL\n AND e.script_hash IS NULL))\n JOIN dbt_node p ON p.workspace_id = e.workspace_id\n AND p.script_path = e.script_path\n AND p.script_hash IS NOT DISTINCT FROM e.script_hash\n AND p.job_id = e.job_id\n AND p.unique_id = e.parent_unique_id\n JOIN dbt_node c ON c.workspace_id = e.workspace_id\n AND c.script_path = e.script_path\n AND c.script_hash IS NOT DISTINCT FROM e.script_hash\n AND c.job_id = e.job_id\n AND c.unique_id = e.child_unique_id\n WHERE e.lineage_kind IN ('copy', 'mod')\n AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "from_path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "from_column!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "to_path!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "to_column!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array", + "UuidArray", + "Text" + ] + }, + "nullable": [ + true, + false, + true, + false, + false + ] + }, + "hash": "55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d" +} diff --git a/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json new file mode 100644 index 0000000000..822695a6c9 --- /dev/null +++ b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms)\n VALUES ($1, $2, $2, '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d" +} diff --git a/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json new file mode 100644 index 0000000000..b2a0d728cd --- /dev/null +++ b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false AND language = 'dbt'\n AND (hash = $3 OR $3 = ANY(parent_hashes))\n FOR SHARE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147" +} diff --git a/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json new file mode 100644 index 0000000000..f0d9aabea3 --- /dev/null +++ b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\",\n EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\",\n EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\"\n FROM unnest($1::text[]) AS c(id)\n FULL JOIN (\n SELECT DISTINCT ON (id) id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2)\n ORDER BY id, created_at DESC\n ) r ON r.id = c.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "value?", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "at?", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "age?", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Float8" + ] + }, + "nullable": [ + true, + true, + true, + true + ] + }, + "hash": "58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b" +} diff --git a/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json new file mode 100644 index 0000000000..09efdc7340 --- /dev/null +++ b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521" +} diff --git a/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json new file mode 100644 index 0000000000..8b04c9e258 --- /dev/null +++ b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10" +} diff --git a/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json b/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json new file mode 100644 index 0000000000..53a3b04d6b --- /dev/null +++ b/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET ci_test_job_ids = NULL, tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c" +} diff --git a/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json b/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json new file mode 100644 index 0000000000..f50b4b743f --- /dev/null +++ b/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific\n WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878" +} diff --git a/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json b/backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json similarity index 63% rename from backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json rename to backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json index 89a02933e0..d234750140 100644 --- a/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json +++ b/backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", + "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary,\n enabled\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", "describe": { "columns": [], "parameters": { @@ -23,10 +23,11 @@ "Bool", "Varchar", "Jsonb", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e" + "hash": "64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63" } diff --git a/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json new file mode 100644 index 0000000000..dc1f6ffbb1 --- /dev/null +++ b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2" +} diff --git a/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json new file mode 100644 index 0000000000..07c923620a --- /dev/null +++ b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7" +} diff --git a/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json new file mode 100644 index 0000000000..b1b92ecc75 --- /dev/null +++ b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "resource_type", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0" +} diff --git a/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json b/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json new file mode 100644 index 0000000000..427ba00075 --- /dev/null +++ b/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET ci_test_job_ids = $5\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385" +} diff --git a/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json new file mode 100644 index 0000000000..3a67641732 --- /dev/null +++ b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 6000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837" +} diff --git a/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json new file mode 100644 index 0000000000..7c11dc80c0 --- /dev/null +++ b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232" +} diff --git a/backend/.sqlx/query-76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477.json b/backend/.sqlx/query-76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477.json new file mode 100644 index 0000000000..ebd492abf3 --- /dev/null +++ b/backend/.sqlx/query-76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET guest_access_enabled = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477" +} diff --git a/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json new file mode 100644 index 0000000000..8eac288a1b --- /dev/null +++ b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH running AS (\n SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag\n )\n SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\",\n (SELECT count(*) FROM worker_ping w\n WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag]\n ) AS \"workers!\"\n FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag)\n LEFT JOIN running r ON r.tag = t.tag\n ORDER BY t.tag", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "running!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "workers!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438" +} diff --git a/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json b/backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json similarity index 78% rename from backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json rename to backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json index e74bd5de0e..f504d7a809 100644 --- a/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json +++ b/backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007" + "hash": "77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1" } diff --git a/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json b/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json new file mode 100644 index 0000000000..98aa0d9367 --- /dev/null +++ b/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text))\n WHERE policy->>'on_behalf_of' = $2 AND policy->>'on_behalf_of_email' = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f" +} diff --git a/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json b/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json new file mode 100644 index 0000000000..b7cccbfad4 --- /dev/null +++ b/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check SET github_posted = true\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2\n AND check_run_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51" +} diff --git a/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json new file mode 100644 index 0000000000..ae8cc698b9 --- /dev/null +++ b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag AS \"tag!\", count AS \"count!\",\n EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\",\n EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\"\n FROM (\n SELECT tag, sum(n)::bigint AS count,\n (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head\n FROM (\n SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head\n FROM v2_job_queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag, priority\n ) g\n GROUP BY tag\n ) t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "delay!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "head_since!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null, + null + ] + }, + "hash": "7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5" +} diff --git a/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json new file mode 100644 index 0000000000..c5740cfb88 --- /dev/null +++ b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_environment_state SET script_path = $3\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba" +} diff --git a/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json new file mode 100644 index 0000000000..f4bd63c718 --- /dev/null +++ b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630" +} diff --git a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json b/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json deleted file mode 100644 index f98f3fdeea..0000000000 --- a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92" -} diff --git a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json b/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json deleted file mode 100644 index 124a6e36c5..0000000000 --- a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value) VALUES ($1, $2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15" -} diff --git a/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json new file mode 100644 index 0000000000..a498ecccd0 --- /dev/null +++ b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912" +} diff --git a/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json b/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json new file mode 100644 index 0000000000..e0d369fd12 --- /dev/null +++ b/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET concluded = true, conclusion = $3, concluded_at = now()\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2 AND NOT concluded\n RETURNING check_run_id, poster_workspace_id, repo_url", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "check_run_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "repo_url", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + false, + false + ] + }, + "hash": "89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b" +} diff --git a/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json b/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json new file mode 100644 index 0000000000..db3362bf0c --- /dev/null +++ b/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ci_test_job_ids\n FROM git_sync_synced_head\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ci_test_job_ids", + "type_info": "UuidArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f" +} diff --git a/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json b/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json new file mode 100644 index 0000000000..14a20fa112 --- /dev/null +++ b/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325" +} diff --git a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json deleted file mode 100644 index 823a65cda3..0000000000 --- a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47" -} diff --git a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json b/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json deleted file mode 100644 index 733478e0e9..0000000000 --- a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7" -} diff --git a/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json new file mode 100644 index 0000000000..b98a71fb8a --- /dev/null +++ b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26" +} diff --git a/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json b/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json new file mode 100644 index 0000000000..b63a9d294e --- /dev/null +++ b/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039" +} diff --git a/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json b/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json new file mode 100644 index 0000000000..a4a692fd4b --- /dev/null +++ b/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb('u/' || $1))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a" +} diff --git a/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json b/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json new file mode 100644 index 0000000000..9960ba85c9 --- /dev/null +++ b/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd" +} diff --git a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json b/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json new file mode 100644 index 0000000000..e751b8fc7c --- /dev/null +++ b/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c" +} diff --git a/backend/.sqlx/query-9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0.json b/backend/.sqlx/query-9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0.json new file mode 100644 index 0000000000..30216f04e9 --- /dev/null +++ b/backend/.sqlx/query-9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE av.raw_app = false)::BIGINT AS \"low_code!\",\n COUNT(*) FILTER (WHERE av.raw_app = true)::BIGINT AS \"raw!\"\n FROM app a\n JOIN app_version av ON av.id = a.versions[array_upper(a.versions, 1)]\n WHERE a.policy->>'sandbox' = 'true'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "low_code!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "raw!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0" +} diff --git a/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json new file mode 100644 index 0000000000..6ef7b20a4f --- /dev/null +++ b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e" +} diff --git a/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json new file mode 100644 index 0000000000..49d01c2b8b --- /dev/null +++ b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders',\n 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders',\n 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82" +} diff --git a/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json new file mode 100644 index 0000000000..909e6ad42d --- /dev/null +++ b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247" +} diff --git a/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json new file mode 100644 index 0000000000..f4c4fd31de --- /dev/null +++ b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text", + "Varchar", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4" +} diff --git a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json b/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json new file mode 100644 index 0000000000..3ed4f5a316 --- /dev/null +++ b/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a" +} diff --git a/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json b/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json similarity index 68% rename from backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json rename to backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json index 5889ab4ee7..0f43afed39 100644 --- a/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json +++ b/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", + "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary,\n nt.enabled\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -102,8 +107,9 @@ true, false, false, - true + true, + false ] }, - "hash": "b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48" + "hash": "a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a" } diff --git a/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json b/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json new file mode 100644 index 0000000000..5f2bb07e24 --- /dev/null +++ b/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT enabled\n FROM native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3" +} diff --git a/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json new file mode 100644 index 0000000000..8c6dd010af --- /dev/null +++ b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH queue_metrics as (\n SELECT id, created_at,\n CASE WHEN jsonb_typeof(value) = 'object'\n THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric)\n ELSE value\n END AS value\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "values!", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16" +} diff --git a/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json deleted file mode 100644 index 7a45e6c402..0000000000 --- a/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5" -} diff --git a/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json b/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json new file mode 100644 index 0000000000..fa43d27ed8 --- /dev/null +++ b/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH survivors AS (\n SELECT value::text AS rendered FROM resource\n WHERE workspace_id = $1 AND NOT (path = ANY($2::text[]))\n )\n SELECT v.path FROM unnest($3::text[]) AS v(path)\n WHERE EXISTS (\n SELECT 1 FROM survivors s\n WHERE strpos(s.rendered, '\"$var:' || v.path || '\"') > 0\n OR strpos(s.rendered, '\"$jsonvar:' || v.path || '\"') > 0\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d" +} diff --git a/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json b/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json new file mode 100644 index 0000000000..26301c39b2 --- /dev/null +++ b/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM git_sync_ci_test_check c\n WHERE c.concluded AND c.github_posted\n AND c.concluded_at < now() - make_interval(secs => $1)\n AND NOT EXISTS (\n SELECT 1 FROM git_sync_synced_head h\n WHERE h.workspace_id = c.workspace_id\n AND h.repo_resource_path = c.repo_resource_path\n AND h.sha = c.head_sha\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [] + }, + "hash": "a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1" +} diff --git a/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json new file mode 100644 index 0000000000..313dec3743 --- /dev/null +++ b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock',\n 'model', 'stock', 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily',\n 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c" +} diff --git a/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json b/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json new file mode 100644 index 0000000000..258ee9eaa7 --- /dev/null +++ b/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT repo_resource_path, head_sha FROM git_sync_ci_test_check\n WHERE workspace_id = $1 AND NOT concluded", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "repo_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "head_sha", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352" +} diff --git a/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json b/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json new file mode 100644 index 0000000000..5a9e5da58e --- /dev/null +++ b/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json @@ -0,0 +1,86 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n workspace_id,\n path,\n route_path,\n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n allowed_origins,\n script_path,\n summary,\n description,\n is_flow,\n mode,\n request_type,\n authentication_method,\n http_method,\n static_asset_config,\n edited_by,\n permissioned_as,\n edited_at,\n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, now(), $21, $22, $23, $24\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598" +} diff --git a/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json new file mode 100644 index 0000000000..2e0e072867 --- /dev/null +++ b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf" +} diff --git a/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json new file mode 100644 index 0000000000..41481e2ec7 --- /dev/null +++ b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f" +} diff --git a/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json new file mode 100644 index 0000000000..222bfc0f82 --- /dev/null +++ b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_unlock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_unlock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385" +} diff --git a/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json b/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json new file mode 100644 index 0000000000..2991d30d1c --- /dev/null +++ b/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET github_posted = github_posted\n AND check_run_id IS NOT DISTINCT FROM GREATEST(check_run_id, $3),\n check_run_id = GREATEST(check_run_id, $3)\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab" +} diff --git a/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json new file mode 100644 index 0000000000..6b2ec01b23 --- /dev/null +++ b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946" +} diff --git a/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json new file mode 100644 index 0000000000..7476ae9871 --- /dev/null +++ b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2\n AND NOT EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64" +} diff --git a/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json new file mode 100644 index 0000000000..934742008f --- /dev/null +++ b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders',\n 'u/a/wh/analytics/raw_orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24" +} diff --git a/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json b/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json new file mode 100644 index 0000000000..31c628c1a8 --- /dev/null +++ b/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM git_sync_ci_test_check\n WHERE poster_workspace_id = $1 AND repo_resource_path = $2 AND head_sha = $3\n ) as \"exists!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff" +} diff --git a/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json new file mode 100644 index 0000000000..980ba7a36c --- /dev/null +++ b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(git_sync, '{repositories}',\n COALESCE((SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' IN ($2, $3)\n THEN CASE WHEN $4::jsonb = 'null'::jsonb\n THEN elem - 'credential'\n ELSE jsonb_set(elem, '{credential}', $4) END\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem), '[]'::jsonb)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n AND EXISTS (\n SELECT 1 FROM jsonb_array_elements(git_sync->'repositories') AS e\n WHERE e->>'git_repo_resource_path' IN ($2, $3)\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c" +} diff --git a/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json b/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json new file mode 100644 index 0000000000..d64cf09687 --- /dev/null +++ b/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET policy = jsonb_set(\n jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)),\n ARRAY['on_behalf_of_email'], to_jsonb($4::text)\n ) WHERE policy->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905" +} diff --git a/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json b/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json new file mode 100644 index 0000000000..ed6a4f9b5e --- /dev/null +++ b/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO http_trigger (\n path, route_path, route_path_key, script_path, is_flow, workspace_id,\n edited_by, edited_at, extra_perms, authentication_method, http_method,\n static_asset_config, is_static_website, workspaced_route, wrap_body,\n raw_string, allowed_origins, authentication_resource_path, summary, description,\n error_handler_path, error_handler_args, retry, request_type, mode,\n permissioned_as, labels\n )\n SELECT\n path, route_path, route_path_key, script_path, is_flow, $1,\n edited_by, edited_at, extra_perms, authentication_method, http_method,\n static_asset_config, is_static_website, workspaced_route, wrap_body,\n raw_string, allowed_origins, authentication_resource_path, summary, description,\n error_handler_path, error_handler_args, retry, request_type, 'disabled'::TRIGGER_MODE,\n permissioned_as, labels\n FROM http_trigger\n WHERE workspace_id = $2\n AND (workspaced_route IS TRUE OR $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1" +} diff --git a/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json b/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json new file mode 100644 index 0000000000..c904dc72fd --- /dev/null +++ b/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as \"count!\"\n FROM git_sync_synced_head h\n JOIN v2_job_completed pc ON pc.id = h.job_id\n JOIN v2_job j ON j.workspace_id = h.workspace_id\n AND j.kind IN ('dependencies', 'flowdependencies', 'appdependencies')\n AND j.created_at >= pc.started_at\n AND j.created_at <= COALESCE(h.tests_dispatched_at, now())\n JOIN v2_job_completed c ON c.id = j.id AND c.status IN ('failure', 'canceled')\n WHERE h.workspace_id = $1 AND h.repo_resource_path = $4\n AND h.branch = $3 AND h.sha = $2\n AND h.source = 'pull'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d" +} diff --git a/backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json b/backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json new file mode 100644 index 0000000000..06b2c50058 --- /dev/null +++ b/backend/.sqlx/query-c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "c73e98e5a937f44724a96ee1b74d31fa71a7be3b8ba3dec9f59f54a6c4030462" +} diff --git a/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json b/backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json similarity index 84% rename from backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json rename to backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json index 0af901b2b8..6a9de7e35b 100644 --- a/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json +++ b/backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary,\n enabled\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -99,8 +104,9 @@ true, false, false, - true + true, + false ] }, - "hash": "15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2" + "hash": "c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1" } diff --git a/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json new file mode 100644 index 0000000000..fee6801a6a --- /dev/null +++ b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n 'u/a/wh/analytics/draft', 'select 3', '{}'),\n ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src',\n 'u/a/wh/analytics/draft_src', 'select 4', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f" +} diff --git a/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json b/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json new file mode 100644 index 0000000000..7bc28f0f39 --- /dev/null +++ b/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json @@ -0,0 +1,185 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n path,\n script_path,\n is_flow,\n route_path,\n authentication_resource_path,\n workspace_id,\n request_type AS \"request_type: _\",\n authentication_method AS \"authentication_method: _\",\n edited_by,\n permissioned_as,\n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n allowed_origins,\n workspaced_route,\n is_static_website,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\",\n mode as \"mode: _\"\n FROM\n http_trigger\n WHERE\n http_method = $1 AND\n (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "authentication_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "request_type: _", + "type_info": { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "authentication_method: _", + "type_info": { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + }, + { + "ordinal": 8, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "static_asset_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "wrap_body", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "raw_string", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "allowed_origins", + "type_info": "TextArray" + }, + { + "ordinal": 14, + "name": "workspaced_route", + "type_info": "Bool" + }, + { + "ordinal": 15, + "name": "is_static_website", + "type_info": "Bool" + }, + { + "ordinal": 16, + "name": "error_handler_path", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "error_handler_args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "retry: _", + "type_info": "Jsonb" + }, + { + "ordinal": 19, + "name": "mode: _", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04" +} diff --git a/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json b/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json new file mode 100644 index 0000000000..52830d5c73 --- /dev/null +++ b/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, email, path, typ, value, created_at)\n VALUES ($1, $2, $3, $4, $5::text::json, COALESCE($8::timestamptz, now()))\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = EXCLUDED.created_at\n WHERE $7::bool = true\n OR $6::timestamptz IS NULL\n OR draft.created_at <= $6::timestamptz\n RETURNING created_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Timestamptz", + "Bool", + "Timestamptz" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7" +} diff --git a/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json new file mode 100644 index 0000000000..58300795af --- /dev/null +++ b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem\n FROM workspace_settings, jsonb_array_elements(git_credentials) AS elem\n WHERE workspace_id = $1 AND elem->>'repo_identity' = $2\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15" +} diff --git a/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json new file mode 100644 index 0000000000..2ef26b366b --- /dev/null +++ b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 100000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba" +} diff --git a/backend/.sqlx/query-cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2.json b/backend/.sqlx/query-cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2.json new file mode 100644 index 0000000000..9ff8b3ace5 --- /dev/null +++ b/backend/.sqlx/query-cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM guest_activity WHERE day < CURRENT_DATE - 60", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2" +} diff --git a/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json b/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json new file mode 100644 index 0000000000..cb87d3b3f6 --- /dev/null +++ b/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args->>'repo_url_resource_path' FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201" +} diff --git a/backend/.sqlx/query-d02c1cfefc7d5f87ca4555a092f5fbc777a91271b80d0e488a8cae79afd56d8f.json b/backend/.sqlx/query-d02c1cfefc7d5f87ca4555a092f5fbc777a91271b80d0e488a8cae79afd56d8f.json new file mode 100644 index 0000000000..f90af3cca7 --- /dev/null +++ b/backend/.sqlx/query-d02c1cfefc7d5f87ca4555a092f5fbc777a91271b80d0e488a8cae79afd56d8f.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg\n FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2\n ORDER BY created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "d02c1cfefc7d5f87ca4555a092f5fbc777a91271b80d0e488a8cae79afd56d8f" +} diff --git a/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json new file mode 100644 index 0000000000..cbcb7f18f7 --- /dev/null +++ b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_lock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749" +} diff --git a/backend/.sqlx/query-a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726.json b/backend/.sqlx/query-d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a.json similarity index 65% rename from backend/.sqlx/query-a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726.json rename to backend/.sqlx/query-d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a.json index d507609d8e..c3134373f6 100644 --- a/backend/.sqlx/query-a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726.json +++ b/backend/.sqlx/query-d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE token SET scopes = $1\n WHERE email = $2 AND token_prefix = $3\n RETURNING token_prefix", + "query": "UPDATE token SET scopes = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR label <> 'guest_session')\n RETURNING token_prefix", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726" + "hash": "d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a" } diff --git a/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json new file mode 100644 index 0000000000..9c75f7ac97 --- /dev/null +++ b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c" +} diff --git a/backend/.sqlx/query-d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590.json b/backend/.sqlx/query-d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590.json new file mode 100644 index 0000000000..ad5a3c9f56 --- /dev/null +++ b/backend/.sqlx/query-d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT usage_kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM asset WHERE kind = 'datatable' AND usage_kind <> 'job'\n GROUP BY 1\n UNION ALL\n SELECT 'job_recent'::text, COUNT(DISTINCT (workspace_id, path))::BIGINT\n FROM asset\n WHERE kind = 'datatable' AND usage_kind = 'job'\n AND created_at > now() - interval '30 days'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590" +} diff --git a/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json new file mode 100644 index 0000000000..3f73c2171e --- /dev/null +++ b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO metrics (id, value)\n SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value)\n FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "JsonbArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3" +} diff --git a/backend/.sqlx/query-d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596.json b/backend/.sqlx/query-d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596.json new file mode 100644 index 0000000000..317f33de07 --- /dev/null +++ b/backend/.sqlx/query-d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value as \"value!: sqlx::types::Json>\",\n created_at\n FROM draft\n WHERE workspace_id = $1\n AND (email = $2 OR email IS NULL)\n AND path = $3\n AND typ = $4\n ORDER BY email NULLS LAST\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + } + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596" +} diff --git a/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json b/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json new file mode 100644 index 0000000000..30cc0e7092 --- /dev/null +++ b/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)))\n WHERE typ IN ('app', 'raw_app')\n AND value->'policy'->>'on_behalf_of' = $2\n AND value->'policy'->>'on_behalf_of_email' = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e" +} diff --git a/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json b/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json new file mode 100644 index 0000000000..70fd38193c --- /dev/null +++ b/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path FROM script WHERE workspace_id = $1 AND deleted = false AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a" +} diff --git a/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json b/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json new file mode 100644 index 0000000000..b0c2d95175 --- /dev/null +++ b/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5" +} diff --git a/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json new file mode 100644 index 0000000000..748ca6d9ec --- /dev/null +++ b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac" +} diff --git a/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json b/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json new file mode 100644 index 0000000000..669e78201d --- /dev/null +++ b/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_admin, operator, email FROM usr where username = $1 AND workspace_id = $2 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc" +} diff --git a/backend/.sqlx/query-dc4a57df3becc610f631ef22c116450390addbfae85fecc61c991d94167e6e99.json b/backend/.sqlx/query-dc4a57df3becc610f631ef22c116450390addbfae85fecc61c991d94167e6e99.json new file mode 100644 index 0000000000..01c0fd19af --- /dev/null +++ b/backend/.sqlx/query-dc4a57df3becc610f631ef22c116450390addbfae85fecc61c991d94167e6e99.json @@ -0,0 +1,226 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n ai_config,\n dbt_warehouses,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts,\n guest_access_enabled,\n guest_jwt_public_key,\n guest_jwt_jwks_url\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_team_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "teams_team_id", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "teams_team_name", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "teams_team_guid", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "slack_name", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "slack_command_script", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "teams_command_script", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "slack_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "customer_id", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "plan", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "webhook", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "ai_config", + "type_info": "Jsonb" + }, + { + "ordinal": 15, + "name": "dbt_warehouses", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 17, + "name": "datatable", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "ducklake", + "type_info": "Jsonb" + }, + { + "ordinal": 19, + "name": "git_sync", + "type_info": "Jsonb" + }, + { + "ordinal": 20, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "default_app", + "type_info": "Varchar" + }, + { + "ordinal": 22, + "name": "default_scripts", + "type_info": "Jsonb" + }, + { + "ordinal": 23, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 24, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 25, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 26, + "name": "git_app_installations", + "type_info": "Jsonb" + }, + { + "ordinal": 27, + "name": "auto_invite", + "type_info": "Jsonb" + }, + { + "ordinal": 28, + "name": "error_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 29, + "name": "success_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "public_app_execution_limit_per_minute", + "type_info": "Int4" + }, + { + "ordinal": 31, + "name": "error_handler_fallback_to_instance_alerts", + "type_info": "Bool" + }, + { + "ordinal": 32, + "name": "guest_access_enabled", + "type_info": "Bool" + }, + { + "ordinal": 33, + "name": "guest_jwt_public_key", + "type_info": "Text" + }, + { + "ordinal": 34, + "name": "guest_jwt_jwks_url", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false, + false, + true, + true + ] + }, + "hash": "dc4a57df3becc610f631ef22c116450390addbfae85fecc61c991d94167e6e99" +} diff --git a/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json new file mode 100644 index 0000000000..9622e63a56 --- /dev/null +++ b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id, ws.git_sync\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id\n WHERE NOT w.deleted\n AND ws.git_sync IS NOT NULL\n AND jsonb_typeof(ws.git_sync->'repositories') = 'array'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b" +} \ No newline at end of file diff --git a/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json b/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json new file mode 100644 index 0000000000..9129b657d3 --- /dev/null +++ b/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as \"count!\" FROM v2_job WHERE id = ANY($1::uuid[])", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8" +} diff --git a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json new file mode 100644 index 0000000000..ae893b6791 --- /dev/null +++ b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int8", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd" +} diff --git a/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json new file mode 100644 index 0000000000..46a9c8e505 --- /dev/null +++ b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570" +} diff --git a/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json new file mode 100644 index 0000000000..797fee956b --- /dev/null +++ b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77" +} diff --git a/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json b/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json new file mode 100644 index 0000000000..cf4d577dbb --- /dev/null +++ b/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, poster_workspace_id, head_sha, repo_url, repo_resource_path,\n check_run_id\n FROM git_sync_ci_test_check\n WHERE NOT concluded OR NOT github_posted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "head_sha", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "repo_url", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "repo_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "check_run_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + true + ] + }, + "hash": "e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619" +} diff --git a/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json b/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json new file mode 100644 index 0000000000..f8c42256bb --- /dev/null +++ b/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM git_sync_synced_head h\n WHERE h.synced_at < now() - make_interval(secs => $1)\n AND EXISTS (\n SELECT 1 FROM git_sync_synced_head n\n WHERE n.workspace_id = h.workspace_id\n AND n.repo_resource_path = h.repo_resource_path\n AND n.branch = h.branch\n AND n.synced_at > h.synced_at\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [] + }, + "hash": "e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c" +} diff --git a/backend/.sqlx/query-e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef.json b/backend/.sqlx/query-e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef.json new file mode 100644 index 0000000000..328465fdf1 --- /dev/null +++ b/backend/.sqlx/query-e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by, canceled_reason,\n (extract(epoch FROM now() - started_at) * 1000)::bigint AS segment_ms\n FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "segment_ms", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true, + null + ] + }, + "hash": "e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef" +} diff --git a/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json b/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json new file mode 100644 index 0000000000..5d0e560ced --- /dev/null +++ b/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(jc.status::text, 'running') as \"status!\"\n FROM unnest($1::uuid[]) AS run(id)\n LEFT JOIN v2_job_completed jc ON jc.id = run.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357" +} diff --git a/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json new file mode 100644 index 0000000000..854ed61857 --- /dev/null +++ b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT n.script_path AS \"script_path!\", n.script_hash, n.job_id AS \"job_id!\"\n FROM dbt_node n\n WHERE n.workspace_id = $1 AND n.asset_path = ANY($2)\n -- The run's snapshot, or the deployed graph when that job stored\n -- none -- a build pins only if it wrote one.\n AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $5)\n THEN $5::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END\n -- The gate, re-decided for every project the walk reaches. That\n -- is what resolving owners in a loop is for: being entitled to\n -- one project is not being entitled to the one that declares a\n -- relation it hands over.\n AND ( $6\n OR n.script_path = ANY($7)\n OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx\n WHERE n.script_path = pfx\n OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) )\n AND CASE\n -- Pinned: which version comes from a job this caller was\n -- already granted, so `script` does not decide THAT -- but\n -- it still decides whether the project may be read, the\n -- same second gate `script_visible` is on the graph. Being\n -- entitled to a run is not being entitled to the SQL\n -- behind it, and column lineage is that SQL's shape. A\n -- version-less row is exempt because it is an editor\n -- buffer, which has no `script` row to ask and reaches\n -- this only through the parse job that wrote it.\n --\n -- One project answers, so a pinned trace never crosses\n -- into another: neither does the graph it annotates.\n WHEN $4::text IS NOT NULL\n THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint\n AND ($3::bigint IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3))\n -- A named version: the deployed one an editor is drawing.\n -- `script` is read under RLS, so this is the visibility\n -- check as well as the existence one. A hash names one\n -- script row, so this arm answers for one project too —\n -- and deliberately: a pin says which stored graph is on\n -- screen, and another project's live graph is not it.\n WHEN $3::bigint IS NOT NULL\n THEN n.script_hash = $3 AND EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3)\n -- Otherwise the version deployed now: an older one's rows\n -- outlive it in `dbt_node` until the sweep, and describe a\n -- project that is no longer what runs. `language` narrows\n -- it the way the graph's own resolution does, so a path\n -- that has since become a script of another kind draws and\n -- explains the same version rather than disagreeing. Read\n -- under RLS, so a project the caller cannot see resolves\n -- to NULL and matches nothing.\n ELSE n.script_hash = (\n SELECT sc.hash FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.language = 'dbt'\n AND sc.deleted = false AND sc.archived = false\n ORDER BY sc.created_at DESC LIMIT 1)\n END", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "job_id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8", + "Text", + "Uuid", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2" +} diff --git a/backend/.sqlx/query-ede15bff96152f209aff756830cbc76b5afa1af6ed324376989117b1054c3447.json b/backend/.sqlx/query-ede15bff96152f209aff756830cbc76b5afa1af6ed324376989117b1054c3447.json new file mode 100644 index 0000000000..1c247ad5b5 --- /dev/null +++ b/backend/.sqlx/query-ede15bff96152f209aff756830cbc76b5afa1af6ed324376989117b1054c3447.json @@ -0,0 +1,82 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n slack_name,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n mute_critical_alerts,\n guest_access_enabled,\n deploy_ui,\n large_file_storage,\n datatable\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_team_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "slack_name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "teams_team_id", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "teams_team_name", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "teams_team_guid", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "guest_access_enabled", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "datatable", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true + ] + }, + "hash": "ede15bff96152f209aff756830cbc76b5afa1af6ed324376989117b1054c3447" +} diff --git a/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json new file mode 100644 index 0000000000..bea6be2a7f --- /dev/null +++ b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, lock_error_logs, created_at)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, $6::text, clock_timestamp()\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text", + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637" +} diff --git a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json new file mode 100644 index 0000000000..5f86fb97d8 --- /dev/null +++ b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed" +} diff --git a/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json b/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json new file mode 100644 index 0000000000..7020c21b0d --- /dev/null +++ b/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO git_sync_synced_head\n (workspace_id, repo_resource_path, branch, sha, source, job_id)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, repo_resource_path, branch, sha)\n DO UPDATE SET source = EXCLUDED.source, job_id = EXCLUDED.job_id, synced_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0" +} diff --git a/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json new file mode 100644 index 0000000000..1a71c74e9e --- /dev/null +++ b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25" +} diff --git a/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json new file mode 100644 index 0000000000..75526d2db8 --- /dev/null +++ b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b" +} diff --git a/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json deleted file mode 100644 index 72acab6120..0000000000 --- a/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20" -} diff --git a/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json new file mode 100644 index 0000000000..1c79a53ab0 --- /dev/null +++ b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14" +} diff --git a/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json b/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json new file mode 100644 index 0000000000..dda976de02 --- /dev/null +++ b/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT repo_url, check_run_id, poster_workspace_id, conclusion,\n created_at, concluded, github_posted, head_ref\n FROM git_sync_ci_test_check\n WHERE workspace_id = $1 AND repo_resource_path = $3 AND head_sha = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "repo_url", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "check_run_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "conclusion", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "concluded", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "github_posted", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "head_ref", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb" +} diff --git a/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json new file mode 100644 index 0000000000..ccb9863853 --- /dev/null +++ b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id',\n 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e" +} diff --git a/backend/.sqlx/query-f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6.json b/backend/.sqlx/query-f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6.json new file mode 100644 index 0000000000..d9b9aabf58 --- /dev/null +++ b/backend/.sqlx/query-f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "guest_jwt_public_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "guest_jwt_jwks_url", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6" +} diff --git a/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json new file mode 100644 index 0000000000..540d2e0e08 --- /dev/null +++ b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2" +} diff --git a/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json b/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json new file mode 100644 index 0000000000..bf8fa238a0 --- /dev/null +++ b/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE native_trigger\n SET enabled = $1\n WHERE\n workspace_id = $2\n AND service_name = $3\n AND external_id = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76" +} diff --git a/backend/.sqlx/query-fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1.json b/backend/.sqlx/query-fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1.json new file mode 100644 index 0000000000..20b5c53392 --- /dev/null +++ b/backend/.sqlx/query-fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET guest_jwt_public_key = $1, guest_jwt_jwks_url = $2 WHERE workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1" +} diff --git a/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json new file mode 100644 index 0000000000..c1eb5fa826 --- /dev/null +++ b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d314168b35..a1a358c4e4 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -430,7 +430,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.14.1", + "indexmap 2.14.2", "lexical-core", "memchr", "num", @@ -781,7 +781,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", "tokio-websockets", "tracing", @@ -873,7 +873,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1348,7 +1348,7 @@ dependencies = [ "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tower 0.5.3", "tracing", ] @@ -1724,7 +1724,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1744,7 +1744,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1787,9 +1787,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -1958,25 +1958,25 @@ dependencies = [ [[package]] name = "bon" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" +checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" dependencies = [ "bon-macros", ] [[package]] name = "bon-macros" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" +checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" dependencies = [ "darling 0.24.1", "ident_case", "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2000,7 +2000,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2095,9 +2095,9 @@ dependencies = [ [[package]] name = "byte-unit" -version = "5.2.5" +version = "5.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a813de7f2bbedb7dce265b64f1cf5908ebe4d56281ece8d847e98113788b9b0" +checksum = "c719d56f7e96194cfc53460976d3ba51c85719747c9c62ed99981847b551152b" dependencies = [ "rust_decimal", "schemars 1.2.2", @@ -2138,13 +2138,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2311,9 +2311,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" dependencies = [ "find-msvc-tools", "jobserver", @@ -2454,7 +2454,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -2733,9 +2733,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] @@ -2763,18 +2763,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2782,27 +2782,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm_winapi" @@ -3047,7 +3047,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -3102,7 +3102,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -3247,7 +3247,7 @@ dependencies = [ "base64 0.22.1", "half", "hashbrown 0.14.5", - "indexmap 2.14.1", + "indexmap 2.14.2", "libc", "log", "object_store", @@ -3426,7 +3426,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.14.1", + "indexmap 2.14.2", "paste", "recursive", "serde_json", @@ -3441,7 +3441,7 @@ checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.14.0", "paste", ] @@ -3595,7 +3595,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.14.0", "log", "recursive", @@ -3618,7 +3618,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.14.0", "log", "paste", @@ -3680,7 +3680,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.14.0", "log", "parking_lot", @@ -3722,7 +3722,7 @@ dependencies = [ "bigdecimal", "datafusion-common", "datafusion-expr", - "indexmap 2.14.1", + "indexmap 2.14.2", "log", "recursive", "regex", @@ -3853,7 +3853,7 @@ dependencies = [ "deno_path_util", "deno_unsync", "futures", - "indexmap 2.14.1", + "indexmap 2.14.2", "libc", "parking_lot", "percent-encoding", @@ -4005,7 +4005,7 @@ dependencies = [ "serde_json", "thiserror 2.0.20", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-socks", "tokio-util", "tokio-vsock", @@ -4122,7 +4122,7 @@ version = "0.228.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf8dbe5abf37d270bb853c5dfe45fbe3b1b6c453877cc11d7fe84e9862a6dbc" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "proc-macro-rules", "proc-macro2", "quote", @@ -4603,7 +4603,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -4850,7 +4850,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -5055,9 +5055,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "fixedbitset" @@ -5077,7 +5077,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "rustc_version 0.4.1", ] @@ -5177,9 +5177,9 @@ dependencies = [ [[package]] name = "frostem" -version = "1.20260821.4" +version = "1.20260821.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481ace7f781f5ae54a5c0a6d6d8edb30adba737cfa1230fbd5632d63ba8dfd80" +checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f" [[package]] name = "fs3" @@ -5319,7 +5319,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -5797,7 +5797,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.14.1", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -5816,7 +5816,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -5898,7 +5898,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "proc-macro2", "quote", "syn 2.0.119", @@ -6194,9 +6194,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] @@ -6262,7 +6262,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tower-service", ] @@ -6328,7 +6328,7 @@ dependencies = [ "rustls 0.23.35", "rustls-native-certs 0.8.4", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tower-service", "webpki-roots 1.0.9", ] @@ -6463,11 +6463,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -6517,6 +6538,8 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -6524,6 +6547,28 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + [[package]] name = "ident_case" version = "1.0.1" @@ -6570,9 +6615,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.1" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -6640,11 +6685,11 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -6664,9 +6709,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "ipnetwork" @@ -6745,9 +6790,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ "defmt", "jiff-core", @@ -6762,18 +6807,19 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" dependencies = [ "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ "jiff-core", "proc-macro2", @@ -6857,9 +6903,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -6958,7 +7004,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", ] [[package]] @@ -7217,7 +7263,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "lazy_static", "libgssapi-sys", @@ -7262,14 +7308,14 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" +checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "plain", - "redox_syscall 0.9.3", + "redox_syscall 0.9.4", ] [[package]] @@ -7516,7 +7562,7 @@ dependencies = [ "rustls-pki-types", "smtp-proto", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "webpki-roots 0.26.11", ] @@ -7741,9 +7787,9 @@ dependencies = [ [[package]] name = "minicov" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3aa3aa12b448ac225b3102217d1ac5cc717908f02722926524b0599c933c7a0" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" dependencies = [ "cc", "walkdir", @@ -7906,7 +7952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "btoi", "byteorder", "bytes", @@ -7965,7 +8011,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -7976,7 +8022,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -7988,7 +8034,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8000,7 +8046,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8125,7 +8171,7 @@ dependencies = [ "dirs-sys 0.4.1", "fancy-regex 0.14.0", "heck", - "indexmap 2.14.1", + "indexmap 2.14.2", "log", "lru 0.12.5", "miette", @@ -8467,7 +8513,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "once_cell", "onig_sys", @@ -8532,7 +8578,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "foreign-types", "libc", @@ -9077,9 +9123,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" dependencies = [ "memchr", "ucd-trie", @@ -9087,9 +9133,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" dependencies = [ "pest", "pest_generator", @@ -9097,9 +9143,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" dependencies = [ "pest", "pest_meta", @@ -9110,9 +9156,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" dependencies = [ "pest", ] @@ -9124,7 +9170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.14.1", + "indexmap 2.14.2", ] [[package]] @@ -9416,9 +9462,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -9485,6 +9531,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -9526,7 +9574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -9544,7 +9592,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit 0.25.15+spec-1.1.0", ] [[package]] @@ -9620,7 +9668,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3ef4f2f0422f23a82ec9f628ea2acd12871c81a9362b02c43c1aa86acfc3ba1" dependencies = [ "futures", - "indexmap 2.14.1", + "indexmap 2.14.2", "nix 0.30.1", "tokio", "tracing", @@ -9633,7 +9681,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "flate2", "hex", @@ -9647,7 +9695,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", ] @@ -9734,7 +9782,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "getopts", "memchr", "unicase", @@ -10028,7 +10076,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10162,16 +10210,16 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] name = "redox_syscall" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d678d17679829e73d371e96880897e98fee2ded7acc0a50bdf8af2affa4b2fe5" +checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10213,7 +10261,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -10311,7 +10359,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", "tower 0.5.3", "tower-http", @@ -10326,11 +10374,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "encoding_rs", "futures-core", @@ -10357,7 +10405,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", "tower 0.5.3", "tower-http", @@ -10378,7 +10426,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.5.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "thiserror 2.0.20", "tower-service", @@ -10396,7 +10444,7 @@ dependencies = [ "getrandom 0.2.17", "http 1.5.0", "hyper 1.11.1", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "retry-policies", "thiserror 2.0.20", @@ -10506,7 +10554,7 @@ dependencies = [ "pastey", "pin-project-lite", "rand 0.10.2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp-macros", "schemars 1.2.2", "serde", @@ -10524,15 +10572,15 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf1c49bd4d52014b94db0877410db273c2008f01628b0252a2e9460ad9b7fda" +checksum = "873b730df6f0a9b74b13eb514e0dca4c2db0d8b68b74af98a2e9bf3f9d436585" dependencies = [ "darling 0.24.1", "proc-macro2", "quote", "serde_json", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -10566,7 +10614,7 @@ dependencies = [ "convert_case 0.10.0", "fnv", "ident_case", - "indexmap 2.14.1", + "indexmap 2.14.2", "proc-macro-crate", "proc-macro2", "quote", @@ -10741,7 +10789,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10754,7 +10802,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", @@ -11169,7 +11217,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.30.0", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11241,7 +11289,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -11254,7 +11302,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -11368,7 +11416,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11390,7 +11438,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11399,7 +11447,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -11444,7 +11492,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -11484,16 +11532,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.1", + "indexmap 2.14.2", "jiff", "schemars 0.9.0", "schemars 1.2.2", @@ -11505,14 +11553,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -11521,7 +11569,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "itoa", "ryu", "serde", @@ -11534,7 +11582,7 @@ version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "itoa", "libyml", "memchr", @@ -11754,9 +11802,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" dependencies = [ "serde", ] @@ -11954,7 +12002,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.14.1", + "indexmap 2.14.2", "log", "memchr", "once_cell", @@ -12017,7 +12065,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "bytes", "chrono", @@ -12061,7 +12109,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "chrono", "crc", @@ -12120,9 +12168,9 @@ dependencies = [ [[package]] name = "sse-stream" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" +checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4" dependencies = [ "bytes", "futures-util", @@ -12311,7 +12359,7 @@ checksum = "72e90b52ee734ded867104612218101722ad87ff4cf74fe30383bd244a533f97" dependencies = [ "anyhow", "bytes-str", - "indexmap 2.14.1", + "indexmap 2.14.2", "serde", "serde_json", "swc_config_macro", @@ -12335,7 +12383,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "is-macro", "num-bigint", "once_cell", @@ -12391,7 +12439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "either", "num-bigint", "phf 0.11.3", @@ -12445,7 +12493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c6f1b8f4232e7a7f614ff7c0f6ccb89c2d028cdf7629f79ad710cff5b28b62c" dependencies = [ "better_scoped_tls", - "indexmap 2.14.1", + "indexmap 2.14.2", "once_cell", "par-core", "phf 0.11.3", @@ -12511,7 +12559,7 @@ checksum = "69ea0052ac23b5b9fbc85bbdb1791b36b918f9d55f594b0ed8e25babb4c32d16" dependencies = [ "base64 0.22.1", "bytes-str", - "indexmap 2.14.1", + "indexmap 2.14.2", "once_cell", "rustc-hash 2.1.3", "serde", @@ -12551,7 +12599,7 @@ version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83259addd99ed4022aa9fc4d39428c008d3d42533769e1a005529da18cde4568" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "num_cpus", "once_cell", "par-core", @@ -12660,9 +12708,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -12724,7 +12772,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "enum-as-inner", "libc", @@ -12752,7 +12800,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -13011,11 +13059,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +checksum = "b81c0cb5fce14f53e49c1d4da0c508334ff12040221bb8ab01b2dabd91d04b6e" dependencies = [ - "unicode-linebreak", + "icu_segmenter", "unicode-width 0.2.2", ] @@ -13056,7 +13104,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -13197,14 +13245,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -13401,9 +13450,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls 0.23.35", "tokio", @@ -13510,7 +13559,7 @@ dependencies = [ "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", ] @@ -13550,7 +13599,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -13559,11 +13608,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.15+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ - "indexmap 2.14.1", + "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -13603,7 +13652,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "socket2 0.5.10", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -13622,6 +13671,7 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", + "flate2", "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", @@ -13635,12 +13685,13 @@ dependencies = [ "rustls-native-certs 0.8.4", "socket2 0.5.10", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-stream", "tower 0.5.3", "tower-layer", "tower-service", "tracing", + "zstd", ] [[package]] @@ -13671,7 +13722,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.1", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper", @@ -13706,7 +13757,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -14022,7 +14073,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -14130,12 +14181,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-normalization" version = "0.1.25" @@ -14335,9 +14380,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14352,7 +14397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "fslock", "gzip-header", "home", @@ -14458,9 +14503,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -14472,9 +14517,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -14482,9 +14527,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -14492,31 +14537,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895a2607575412a4eda1df892084a375ea10dfeadc4d7d2ab87b854e4ddc7ba1" +checksum = "45863ef0bef521c12124eb39d9a38513c47db75f22e503c06beacd40afeb35db" dependencies = [ "async-trait", "cast", @@ -14536,20 +14581,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288cb0ebe215033bf949ae1fd046726daa4c32a157f24b9dc6ac387a52aa759" +checksum = "8c89dcab8b516b6b603baca9d550b7282d68fcc7f367e3956cff7ebf406a3f12" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "wasm-bindgen-test-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff1c1b360982e93b6d8ea9c04836f71dba0817a16f91e229cf3a51bdd9d987" +checksum = "f37b4f992cebe528ef34964ae69681ac0fe7080071e7298e46008f9d380302af" [[package]] name = "wasm-streams" @@ -14603,9 +14648,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -14747,7 +14792,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-nats", @@ -14764,6 +14809,7 @@ dependencies = [ "git-version", "hex", "hmac", + "jsonwebtoken 8.3.0", "lazy_static", "once_cell", "opentelemetry 0.30.0", @@ -14771,7 +14817,7 @@ dependencies = [ "prometheus", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rumqttc", "rustls 0.23.35", "serde", @@ -14791,6 +14837,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-stream", + "tower-cookies", "tracing", "tracing-subscriber", "url", @@ -14802,6 +14849,7 @@ dependencies = [ "windmill-api-client", "windmill-api-scripts", "windmill-api-settings", + "windmill-api-users", "windmill-autoscaling", "windmill-common", "windmill-dep-map", @@ -14832,7 +14880,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.803.0" +version = "1.811.1" dependencies = [ "async-stream", "async-trait", @@ -14848,7 +14896,7 @@ dependencies = [ "http 1.5.0", "lazy_static", "mime_guess", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -14865,7 +14913,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -14878,7 +14926,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "argon2", @@ -14909,7 +14957,7 @@ dependencies = [ "hmac", "http 1.5.0", "hyper 1.11.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -14928,7 +14976,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rsa", "rust-embed", "rustls 0.23.35", @@ -15018,7 +15066,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15041,7 +15089,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15058,7 +15106,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -15068,7 +15116,7 @@ dependencies = [ "jsonwebtoken 8.3.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15084,7 +15132,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.803.0" +version = "1.811.1" dependencies = [ "reqwest 0.12.28", "serde", @@ -15094,7 +15142,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15111,7 +15159,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15133,7 +15181,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -15142,7 +15190,7 @@ dependencies = [ "candle-transformers", "hf-hub", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15156,7 +15204,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15172,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15194,7 +15242,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15215,7 +15263,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15229,7 +15277,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-nats", @@ -15243,7 +15291,7 @@ dependencies = [ "hmac", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "rumqttc", "serde", @@ -15264,7 +15312,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -15289,7 +15337,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15298,7 +15346,7 @@ dependencies = [ "hex", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "semver 1.0.28", "serde", "serde_json", @@ -15317,12 +15365,12 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "axum 0.8.9", "http 1.5.0", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.14.0", "lazy_static", "serde", @@ -15339,7 +15387,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15359,7 +15407,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15370,7 +15418,7 @@ dependencies = [ "lazy_static", "prometheus", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -15397,7 +15445,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -15425,7 +15473,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.803.0" +version = "1.811.1" dependencies = [ "lazy_static", "serde", @@ -15437,7 +15485,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.803.0" +version = "1.811.1" dependencies = [ "argon2", "axum 0.8.9", @@ -15461,7 +15509,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15475,7 +15523,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.803.0" +version = "1.811.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15510,7 +15558,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.803.0" +version = "1.811.1" dependencies = [ "chrono", "lazy_static", @@ -15524,7 +15572,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -15543,7 +15591,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.803.0" +version = "1.811.1" dependencies = [ "aes-gcm", "aho-corasick", @@ -15560,7 +15608,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "chrono", "chrono-tz", @@ -15581,7 +15629,7 @@ dependencies = [ "hex", "hmac", "hyper 1.11.1", - "indexmap 2.14.1", + "indexmap 2.14.2", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -15599,12 +15647,13 @@ dependencies = [ "pep440_rs", "phf 0.11.3", "pin-project-lite", + "pkcs1", "postgres-native-tls 0.5.3", "prometheus", "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "reqwest-retry", "rsa", @@ -15615,6 +15664,7 @@ dependencies = [ "serde_yml", "sha2 0.10.9", "size", + "spki", "sqlx", "strum", "strum_macros", @@ -15647,7 +15697,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.803.0" +version = "1.811.1" dependencies = [ "chrono", "futures", @@ -15667,7 +15717,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.803.0" +version = "1.811.1" dependencies = [ "regex", "serde", @@ -15677,12 +15727,13 @@ dependencies = [ "tracing", "uuid", "windmill-common", + "windmill-dep-map", "windmill-queue", ] [[package]] name = "windmill-indexer" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15709,7 +15760,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "futures", @@ -15726,7 +15777,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.803.0" +version = "1.811.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15742,7 +15793,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -15750,7 +15801,7 @@ dependencies = [ "futures", "http 1.5.0", "oauth2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "serde", "serde_json", @@ -15763,7 +15814,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -15775,7 +15826,7 @@ dependencies = [ "http 1.5.0", "itertools 0.14.0", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -15794,7 +15845,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "arc-swap", @@ -15819,7 +15870,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-stream", @@ -15837,9 +15888,10 @@ dependencies = [ "lazy_static", "object_store", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", + "serial_test", "sqlx", "tempfile", "tokio", @@ -15853,7 +15905,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "futures", @@ -15871,7 +15923,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.803.0" +version = "1.811.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15880,7 +15932,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -15892,7 +15944,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde_json", @@ -15904,7 +15956,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "gosyn", @@ -15916,7 +15968,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -15928,7 +15980,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde_json", @@ -15940,7 +15992,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "nu-parser", @@ -15951,7 +16003,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15962,7 +16014,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15974,7 +16026,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "rustpython-ast", @@ -15985,7 +16037,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-recursion", @@ -16007,7 +16059,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde_json", @@ -16019,7 +16071,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -16033,7 +16085,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16050,7 +16102,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -16063,7 +16115,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde", @@ -16075,7 +16127,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -16093,7 +16145,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16109,7 +16161,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "rustpython-ast", @@ -16125,7 +16177,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -16139,7 +16191,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-recursion", @@ -16160,7 +16212,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "serde_urlencoded", @@ -16178,7 +16230,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "const_format", @@ -16202,7 +16254,7 @@ dependencies = [ "lazy_static", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rustls 0.23.35", "serde", "serde_json", @@ -16218,7 +16270,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.803.0" +version = "1.811.1" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16229,7 +16281,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-recursion", @@ -16244,7 +16296,7 @@ dependencies = [ "lazy_static", "magic-crypt", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16264,7 +16316,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16288,7 +16340,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16299,7 +16351,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -16321,7 +16373,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16348,7 +16400,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16363,7 +16415,7 @@ dependencies = [ "lazy_static", "quick_cache", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16381,7 +16433,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16401,7 +16453,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16416,7 +16468,7 @@ dependencies = [ "jsonwebtoken 8.3.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -16435,7 +16487,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16471,7 +16523,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16494,7 +16546,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16518,7 +16570,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-nats", @@ -16542,7 +16594,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16577,7 +16629,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16605,7 +16657,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-trait", @@ -16630,10 +16682,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", "itertools 0.14.0", @@ -16649,7 +16701,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-once-cell", @@ -16694,6 +16746,7 @@ dependencies = [ "opentelemetry 0.30.0", "opentelemetry-proto 0.30.0", "oracle", + "parquet", "pem 3.0.6", "pep440_rs", "postgres-native-tls 0.5.3", @@ -16704,7 +16757,7 @@ dependencies = [ "rand 0.9.0", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "rsa", "rust_decimal", @@ -16721,7 +16774,7 @@ dependencies = [ "tiberius", "tokio", "tokio-postgres", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-stream", "tokio-util", "tracing", @@ -16766,7 +16819,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.803.0" +version = "1.811.1" dependencies = [ "bytes", "futures", @@ -17017,7 +17070,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "widestring", "windows-sys 0.52.0", ] @@ -17466,18 +17519,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", @@ -17534,6 +17587,7 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -17542,6 +17596,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -17555,7 +17610,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -17565,7 +17620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "crc32fast", - "indexmap 2.14.1", + "indexmap 2.14.2", "memchr", "typed-path", ] @@ -17593,18 +17648,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f432f022c5..d960a0d3e8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.803.0" +version = "1.811.1" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.803.0" +version = "1.811.1" authors = ["Ruben Fiszel "] edition = "2021" @@ -108,6 +108,10 @@ debug = "line-tables-only" [profile.dev.package."*"] debug = false +# The expression parser's fallback call chain exhausts worker stacks without optimization. +[profile.dev.package.php-parser-rs] +opt-level = 1 + [profile.release] lto = "thin" debug = "line-tables-only" @@ -351,6 +355,8 @@ windmill-trigger-sqs.workspace = true windmill-trigger-gcp.workspace = true windmill-trigger-azure.workspace = true windmill-api-auth.workspace = true +tower-cookies.workspace = true +windmill-api-users.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true @@ -365,6 +371,7 @@ aws-config.workspace = true aws-credential-types.workspace = true hmac.workspace = true hex.workspace = true +jsonwebtoken = { workspace = true } [workspace.dependencies] @@ -597,6 +604,8 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] } const-str = "0.5" constant_time_eq = "0.3.1" rsa = "^0" +spki = { version = "0.7", features = ["pem"] } +pkcs1 = "0.7" aes-gcm = "0.10.3" async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" @@ -660,6 +669,12 @@ process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" datafusion = "47.0.0" +# The row API only: a dbt engine's parquet index is six string columns, so this +# needs no arrow and no writer. `parquet` is already in the tree with `arrow` for +# every shipped edition (`oss_core`), and cargo unifies the features there; this +# set is what a build WITHOUT object storage compiles. ZSTD is what the engine +# writes today, snap what parquet writers most often default to. +parquet = { version = "55.2.0", default-features = false, features = ["snap", "zstd"] } object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" @@ -686,7 +701,7 @@ async-stream = "^0" opentelemetry = "0.30.0" tracing-opentelemetry = "0.31.0" opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] } -opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto"] } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto", "gzip-tonic", "zstd-tonic"] } opentelemetry-appender-tracing = "0.30.0" opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] } opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] } diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index f5725d65a3..af33877078 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -104,7 +104,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | T6 | Disclosure of secrets, resource credentials, and workspace encryption keys across the authorization boundary (AI proxy, MCP, caches, export); database read additionally yields plaintext instance-level `global_settings` secrets | remote_auth | EP6, EP14, EP13 | Secret variables, encryption keys, resource creds, global settings | critical | likely | partially_mitigated | RLS on `$var:`, cache scoping by caller, admin checks on export; per-workspace secret *variables* encrypted at rest, but `global_settings` is plaintext under the default DB secret backend | GHSA-jwg4-v3cj-rvfm, GHSA-8m2p-2crh-9h3w, GHSA-6635-6fch-v8px, GHSA-437f-725p-7w84, GHSA-f27g-j463-q85w (CVE-2026-26964), GHSA-j679-v6vj-jfxc, GHSA-6vrr-fq33-qpfp, 0ba128afe7, 7836a4e733, ff8e39c69b | | T7 | Full instance compromise from insecure deployment defaults (dind control, default admin/`changeme`, exposed Postgres, publicly readable SUPERADMIN_SECRET) | remote_unauth | EP15 | All assets | critical | likely | partially_mitigated | first-time-setup warning on default admin; docs recommend hardening | GHSA-3vpp-vf62-wqp6, GHSA-24fr-44f8-fqwg (CVE-2026-29059), GHSA-6q36-5p3h-766j | | T8 | Unauthenticated RCE via the Debugger WebSocket: `/ws_debug/*` exposed by the gateway/ingress with the debugger service as the auth boundary; signature gate was bypassable via `program`-mode launches (read+exec an arbitrary server-side file path, never signed) even with signing on, and the WS handshake had no Origin check (CSWSH) | remote_unauth | EP15 | Worker host, all assets | critical | possible | partially_mitigated | `program`-mode launches now rejected when `REQUIRE_SIGNED_DEBUG_REQUESTS` is on (signing covers every launch, not just inline `code`); shipped `docker-compose` now defaults `REQUIRE_SIGNED_DEBUG_REQUESTS=true`; opt-in `DEBUG_ALLOWED_ORIGINS` allowlist rejects cross-origin handshakes. Residual: code default is secure but operators can still set `=false`; origin allowlist is opt-in | GHSA-725h-99vx-9xr4 | -| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | +| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override; release images (`v*` tags) keyless-signed with cosign, with per-platform SPDX SBOMs embedded at build time (covered by the signed index digest) + SLSA provenance (GitHub artifact attestations) | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | | T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | | T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, S3 download content-type, or a script-chosen `text/html` content type on `run_wait_result` / sync HTTP-route responses (GET-reachable with the `SameSite=Lax` session cookie) | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads and on every `result_to_response` composite result (inserted after `wm_headers`; hop-by-hop names such as `Connection` rejected so a proxy cannot strip them) | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0, WIN-2471 | | T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | @@ -169,4 +169,4 @@ check. | Canonicalize + confine all file-path inputs to a base dir and never follow symlinks in log/file readers | T13 | yes | S | | Mask secrets at the log sink and keep secrets out of worker process env (`/proc`) — pass via files/pipes scrubbed after use | T15 | partial | M | | Add global rate limiting and per-tenant resource/queue quotas at the edge | T16, T18 | partial | M | -| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release | T9 | partial | M | +| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release — release images now cosign-signed with SBOM + SLSA provenance attestations; remaining: CI action SHA-pinning, hub-script integrity, rhel/rpi images | T9 | partial | M | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9f37cbedb0..9f9388b41a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -f5b783d2f7608e1ff3a817caa8b719e06f8b8981 +a4da009a5eae72bd55f34de41ba7929b53d53c9b diff --git a/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql b/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql new file mode 100644 index 0000000000..b21e2bb474 --- /dev/null +++ b/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS git_sync_ci_test_check; diff --git a/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql b/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql new file mode 100644 index 0000000000..59fabb890c --- /dev/null +++ b/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql @@ -0,0 +1,39 @@ +-- One "Windmill CI tests" GitHub check run per (fork workspace, repository, PR head commit): +-- the pull_request webhook opens the check in_progress and it is concluded once +-- the fork's CI tests settle, so the results can gate a GitHub PR. +CREATE TABLE git_sync_ci_test_check ( + -- The fork workspace whose CI tests gate the PR: keys the row, and its `ci_test` + -- jobs are what the check reflects. + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + head_sha VARCHAR(64) NOT NULL, + -- The PR's head branch: the check waits until the fork's synced state for this + -- branch (written by its pushes and pulls alike) names `head_sha`. + head_ref VARCHAR(255) NOT NULL, + -- The workspace whose git host credential posts the check: the one that received + -- the pull request webhook (the parent owning the repo hook). + poster_workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + repo_url TEXT NOT NULL, + -- The fork's copy of the repository resource: keys the synced-head lookup, since a + -- fork syncing two repositories names its branch identically in both. + repo_resource_path VARCHAR(255) NOT NULL, + -- NULL when the GitHub check-run creation failed; the poller retries the create. + check_run_id BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + concluded BOOLEAN NOT NULL DEFAULT false, + conclusion TEXT, + concluded_at TIMESTAMPTZ, + -- Decoupled from `concluded` so a failed check-run PATCH is retried by the + -- poller instead of hanging a required check on GitHub. + github_posted BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (workspace_id, repo_resource_path, head_sha) +); + +-- Rows still needing action (create retry, conclusion, timeout, delivery retry). +-- A row drops out only once it is both concluded and delivered to GitHub, so the +-- per-job conclusion hook and the poller sweeper both scan a small live set. +CREATE INDEX idx_git_sync_ci_test_check_pending + ON git_sync_ci_test_check (workspace_id) + WHERE NOT concluded OR NOT github_posted; + +GRANT ALL ON git_sync_ci_test_check TO windmill_user; +GRANT ALL ON git_sync_ci_test_check TO windmill_admin; diff --git a/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql b/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql new file mode 100644 index 0000000000..bee3e6034c --- /dev/null +++ b/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE http_trigger DROP COLUMN allowed_origins; diff --git a/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql b/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql new file mode 100644 index 0000000000..444fc46535 --- /dev/null +++ b/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE http_trigger ADD COLUMN allowed_origins TEXT[]; diff --git a/backend/migrations/20260901192755_guest_app_access.down.sql b/backend/migrations/20260901192755_guest_app_access.down.sql new file mode 100644 index 0000000000..065e60c0c8 --- /dev/null +++ b/backend/migrations/20260901192755_guest_app_access.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS guest_activity; +ALTER TABLE workspace_settings DROP COLUMN guest_access_enabled; diff --git a/backend/migrations/20260901192755_guest_app_access.up.sql b/backend/migrations/20260901192755_guest_app_access.up.sql new file mode 100644 index 0000000000..a29582ba99 --- /dev/null +++ b/backend/migrations/20260901192755_guest_app_access.up.sql @@ -0,0 +1,28 @@ +-- Guest app access: a workspace-level switch, off by default. An app whose policy says +-- `execution_mode: guest` only admits guests where this is on, and the check runs where +-- the guest session is minted -- an app definition carries its policy, so git-sync and +-- the CLI push `guest` past every UI gate. +ALTER TABLE workspace_settings + ADD COLUMN guest_access_enabled BOOLEAN NOT NULL DEFAULT false; + +-- A guest leaves no `usr` or `password` row, which is what keeps them off every seat +-- counter, so this is the only durable record that one was here: a row per guest, +-- workspace and day, written when the session is minted. +-- +-- Deliberately not the audit log. The seat scan is served by a partial index whose +-- predicate names the login operations literally, and `audit_partitioned` is a +-- partitioned table, where `CREATE INDEX CONCURRENTLY` is unsupported -- adding a +-- guest operation to that predicate means a locking rebuild on the largest table an +-- instance has. Guest logins still write `users.login_guest` for the audit trail; +-- nothing counts them from there. +CREATE TABLE guest_activity ( + email VARCHAR(255) NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + day DATE NOT NULL DEFAULT CURRENT_DATE, + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (email, workspace_id, day) +); + +-- The retention delete filters on day alone; the PK only reaches it through two +-- other columns. +CREATE INDEX idx_guest_activity_day ON guest_activity (day); diff --git a/backend/migrations/20260903071242_guest_jwt_entry.down.sql b/backend/migrations/20260903071242_guest_jwt_entry.down.sql new file mode 100644 index 0000000000..f3e758819b --- /dev/null +++ b/backend/migrations/20260903071242_guest_jwt_entry.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE guest_activity DROP COLUMN jwt_entry; +ALTER TABLE workspace_settings + DROP CONSTRAINT workspace_settings_guest_jwt_one_key, + DROP COLUMN guest_jwt_public_key, + DROP COLUMN guest_jwt_jwks_url; diff --git a/backend/migrations/20260903071242_guest_jwt_entry.up.sql b/backend/migrations/20260903071242_guest_jwt_entry.up.sql new file mode 100644 index 0000000000..0cc449f539 --- /dev/null +++ b/backend/migrations/20260903071242_guest_jwt_entry.up.sql @@ -0,0 +1,15 @@ +-- A second way in for a guest: a JWT minted by the embedding customer's own backend and +-- verified against a key the workspace admin configured. One key shape per workspace, +-- a PEM public key or a JWKS URL, never both: a token is verified against exactly one +-- source, and two would make "which one refused it" undiagnosable. +ALTER TABLE workspace_settings + ADD COLUMN guest_jwt_public_key TEXT, + ADD COLUMN guest_jwt_jwks_url TEXT, + ADD CONSTRAINT workspace_settings_guest_jwt_one_key + CHECK (guest_jwt_public_key IS NULL OR guest_jwt_jwks_url IS NULL); + +-- Whether the guest came in on a JWT that day (as opposed to, or as well as, an +-- identity-provider sign-in). The seat telemetry reports the two entries apart, since +-- an app-only user routed through a guest JWT is one that `jwt_ext_` would have counted. +ALTER TABLE guest_activity + ADD COLUMN jwt_entry BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql new file mode 100644 index 0000000000..ca88ec0f1e --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS git_credentials; diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql new file mode 100644 index 0000000000..7ece575cc1 --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql @@ -0,0 +1,8 @@ +-- Server-owned git-sync credentials, one entry per repository the workspace +-- holds a token for, keyed by that repository rather than by a resource naming +-- it: a resource's URL is writable, and its path is not settled while it is +-- being created. +-- Kept out of `git_sync` because that column is copied into forks and returned +-- by the workspace settings API; this one is copied by neither. +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS git_credentials JSONB NOT NULL DEFAULT '[]'::jsonb; diff --git a/backend/migrations/20260904135713_dbt_environment_state.down.sql b/backend/migrations/20260904135713_dbt_environment_state.down.sql new file mode 100644 index 0000000000..dab9025417 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS dbt_environment_state; diff --git a/backend/migrations/20260904135713_dbt_environment_state.up.sql b/backend/migrations/20260904135713_dbt_environment_state.up.sql new file mode 100644 index 0000000000..a49df5b3d1 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.up.sql @@ -0,0 +1,52 @@ +-- The dbt state one project last built into one environment: the `manifest.json` +-- (and the `run_results.json` beside it) that `dbt --defer --state ` resolves +-- an unbuilt `ref()` through. +-- +-- Separate from `dbt_run_state`, which answers a different question. That one is +-- keyed by the executing principal and holds the LAST run whatever its outcome, +-- so `dbt retry` can resume its failures; this one is keyed by environment and +-- holds the last SUCCESSFUL run, because a relation a later run defers to has to +-- exist. Merging them would make a retry resume a run that is not the last one, +-- or a deferral point at relations a failed run never wrote. +CREATE TABLE IF NOT EXISTS dbt_environment_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + -- The workspace warehouse, the dbt target, and the database and schema that + -- target resolves to. All four, because deferring is resolving a relation + -- NAME: a repointed warehouse or a moved schema makes the stored manifest + -- describe relations that are not where this run would look for them, and the + -- run has no other way to notice. A move therefore reads as an environment + -- with no state yet rather than as state that silently no longer fits. + -- + -- TEXT rather than VARCHAR(255): a project bringing its own `profiles.yml` + -- spells its own schema and database, so the length is the project's. + environment TEXT NOT NULL, + -- The run that published it, so a deferring run can say what it deferred to. + job_id UUID NOT NULL, + -- Exactly one home each. A manifest grows with the project and passes a few + -- hundred KB on a handful of models, so a large one goes to the INSTANCE's + -- object storage and this row keeps the key; a small one stays here, where it + -- costs no round trip and works on an instance that has configured no storage + -- at all. The instance's and not the workspace's, because a member can write + -- the workspace bucket under a key of their choosing, and a manifest is what a + -- later run resolves every unbuilt `ref()` through. `run_results.json` is a + -- tenth of the size and takes the same two homes rather than a rule of its own. + manifest TEXT, + manifest_key TEXT, + run_results TEXT, + run_results_key TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, environment), + CONSTRAINT dbt_environment_state_manifest_one_home + CHECK (num_nonnulls(manifest, manifest_key) = 1), + CONSTRAINT dbt_environment_state_run_results_one_home + CHECK (num_nonnulls(run_results, run_results_key) <= 1) +); + +-- No age sweep, unlike the per-run graph rows next door: this table holds one +-- row per script per environment and replaces it in place, so it does not grow +-- with runs, and its reader is every later run of that script — a project that +-- runs monthly must still find last month's state. It goes with the script +-- instead, alongside `dbt_run_state`. +GRANT ALL ON dbt_environment_state TO windmill_user; +GRANT ALL ON dbt_environment_state TO windmill_admin; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.down.sql b/backend/migrations/20260904143633_dbt_column_lineage.down.sql new file mode 100644 index 0000000000..a053a94291 --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS dbt_column_edge; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS column_schema; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.up.sql b/backend/migrations/20260904143633_dbt_column_lineage.up.sql new file mode 100644 index 0000000000..cb463c249f --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.up.sql @@ -0,0 +1,68 @@ +-- Column-level lineage, from the engine's own static analysis. +-- +-- `manifest.json` carries none, which is why decision 14 recorded the feature as +-- unavailable. The edges exist in a different artifact: an engine that does +-- static analysis writes `target/index/dbt.column_lineage.parquet` under +-- `dbt compile --static-analysis strict --write-index`. That pass is opt-in per +-- project (`column_lineage: true`), because strict analysis rejects SQL the +-- default accepts and must never become a silent requirement of running a build. + +-- One column-to-column edge, keyed exactly like `dbt_edge`: a version's graph +-- dies with its version through the composite foreign key, a run's snapshot is +-- keyed by `job_id` with the zero UUID meaning "the version's own graph", and an +-- editor buffer's parse carries a NULL `script_hash` keyed to its preview job. +CREATE TABLE IF NOT EXISTS dbt_column_edge ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + parent_unique_id TEXT NOT NULL, + parent_column TEXT NOT NULL, + child_unique_id TEXT NOT NULL, + child_column TEXT NOT NULL, + -- dbt's own word for how the value travelled: `copy` (passthrough), `mod` + -- (transformed), `scan` (the column was read to produce the ROW rather than + -- the value -- a join key, a `where` predicate, a `group by`). TEXT rather + -- than an enum because the engine treats the set as open: its own reader maps + -- those three and returns anything else verbatim. + lineage_kind TEXT NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Two partial unique indexes rather than a primary key, for the reason + -- 20260801121717 gives: a versioned graph is keyed by its version, a buffer + -- parse by its job alone. `lineage_kind` is part of the key because it is part + -- of the fact: a column that is both projected and used as a predicate for the + -- same output column has a `copy` edge AND a `scan` one, and the digest counts + -- both. Leaving it out let `ON CONFLICT DO NOTHING` drop the second while the + -- digest still claimed it was stored. + CONSTRAINT dbt_column_edge_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_versioned_key + ON dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_editor_key + ON dbt_column_edge (workspace_id, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NULL; + +-- Same age sweep as the other per-run rows, and the same reason there is no +-- foreign key to `v2_job`. +CREATE INDEX IF NOT EXISTS idx_dbt_column_edge_run_age ON dbt_column_edge (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; + +-- The real column schema of a node, which only static analysis knows: an +-- ordered `[{"name": …, "type": …}]`, from `dbt.node_columns.parquet`. +-- +-- Beside `columns` rather than folded into it. `columns` is the DECLARED +-- metadata `manifest.json` carries -- the names an author wrote in `schema.yml` +-- and the prose against them -- and stays exactly that, so a project that +-- documents two of forty columns keeps saying so. This is the other forty, +-- typed, in the order the model produces them. +ALTER TABLE dbt_node ADD COLUMN IF NOT EXISTS column_schema JSONB; + +GRANT ALL ON dbt_column_edge TO windmill_user; +GRANT ALL ON dbt_column_edge TO windmill_admin; diff --git a/backend/migrations/20260908113909_native_trigger_enabled.down.sql b/backend/migrations/20260908113909_native_trigger_enabled.down.sql new file mode 100644 index 0000000000..75f26b22c4 --- /dev/null +++ b/backend/migrations/20260908113909_native_trigger_enabled.down.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger DROP COLUMN IF EXISTS enabled; diff --git a/backend/migrations/20260908113909_native_trigger_enabled.up.sql b/backend/migrations/20260908113909_native_trigger_enabled.up.sql new file mode 100644 index 0000000000..15139498d7 --- /dev/null +++ b/backend/migrations/20260908113909_native_trigger_enabled.up.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT true; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.down.sql b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql new file mode 100644 index 0000000000..32ae30d2bd --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql @@ -0,0 +1,9 @@ +-- Dropping the column drops the primary key and the identity sequence with it, and +-- only marks the column dropped in the catalog rather than rewriting the table, so +-- this takes the ACCESS EXCLUSIVE lock but not the time. + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_column_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_graph_snapshot DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.up.sql b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql new file mode 100644 index 0000000000..399fe6cc5f --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql @@ -0,0 +1,31 @@ +-- These five had neither a PRIMARY KEY nor an explicit REPLICA IDENTITY, which makes +-- PostgreSQL reject UPDATE and DELETE on them under logical replication. +-- `deployment_metadata` and `metrics` are the other two, one migration each after this. +-- +-- The surrogate cannot be swapped for a natural key: every unique index on all five is +-- PARTIAL, split on `script_hash IS NULL`, and a partial index cannot back a primary +-- key. The partial uniques stay; they are what the ON CONFLICT clauses infer. +-- +-- Each ALTER rewrites its table under ACCESS EXCLUSIVE and holds it unavailable for +-- the rewrite. These five share a transaction because each is bounded by what a +-- workspace holds rather than by how long it has run, so none can grow into the one +-- that locks the rest; a transaction holds all its locks until it commits. +-- +-- An instance that cannot afford that lock at startup can set REPLICA IDENTITY FULL +-- on these tables instead, which unblocks replication by itself, and run these +-- idempotent ALTERs in a maintenance window first. + +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_node + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_column_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_graph_snapshot + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql new file mode 100644 index 0000000000..3ce4ae52a6 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE deployment_metadata DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql new file mode 100644 index 0000000000..951253ce64 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql @@ -0,0 +1,13 @@ +-- The sixth of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the one table in the set with no retention +-- sweep -- rows accumulate per deployed script hash, flow version and app version for +-- the life of the instance -- so on an instance that upgrades after years of deploys +-- its ACCESS EXCLUSIVE rewrite is the one that could hold the others locked. +-- +-- No natural key: each row is a script, flow OR app deployment, and the three unique +-- indexes are partial on exactly that split, so none of them covers every row. + +ALTER TABLE deployment_metadata + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.down.sql b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql new file mode 100644 index 0000000000..1ad428dc02 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE metrics DROP COLUMN IF EXISTS row_id; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.up.sql b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql new file mode 100644 index 0000000000..b863035990 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql @@ -0,0 +1,12 @@ +-- The last of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the largest (~400 MB / 750k rows on the +-- instance this was measured on, a steady state: `queue_%` rows, which are nearly all +-- of them, are swept at 14 days) and its ALTER rewrites it under ACCESS EXCLUSIVE. One +-- migration is one transaction, so alone it holds no lock on the others as it rewrites. +-- +-- The surrogate cannot be called `id` -- `metrics.id` holds the metric NAME. + +ALTER TABLE metrics + ADD COLUMN IF NOT EXISTS row_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql b/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql new file mode 100644 index 0000000000..a089085444 --- /dev/null +++ b/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS git_sync_synced_head; diff --git a/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql b/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql new file mode 100644 index 0000000000..a0c81d2671 --- /dev/null +++ b/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql @@ -0,0 +1,24 @@ +-- One row per commit a workspace has come to reflect on a branch, written when a +-- pull of that commit succeeds or a deploy push produces it. The "Windmill CI +-- tests" PR check reads it to know when a workspace reflects a PR head, and +-- records the head's CI test runs on it. Kept apart from `workspace_settings.git_sync.auto_pull.last_synced_sha`, +-- which decides whether the next poll pulls and is client-round-tripped settings. +CREATE TABLE git_sync_synced_head ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- Repository resource path without its `$res:` prefix. + repo_resource_path VARCHAR(255) NOT NULL, + branch VARCHAR(255) NOT NULL, + sha VARCHAR(64) NOT NULL, + -- 'pull' rows name the pull job; 'push' rows the deploy push job. + source VARCHAR(4) NOT NULL CHECK (source IN ('pull', 'push')), + job_id UUID, + synced_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- The head's own CI test suite, dispatched once the workspace reflects it and its + -- dependency jobs settled: the "Windmill CI tests" check reads exactly these runs. + tests_dispatched_at TIMESTAMPTZ, + ci_test_job_ids UUID[], + PRIMARY KEY (workspace_id, repo_resource_path, branch, sha) +); + +GRANT ALL ON git_sync_synced_head TO windmill_user; +GRANT ALL ON git_sync_synced_head TO windmill_admin; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql new file mode 100644 index 0000000000..3405d5a681 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql @@ -0,0 +1,7 @@ +DROP INDEX IF EXISTS index_app_version_on_app_id; + +DROP INDEX IF EXISTS index_app_script_on_app; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql new file mode 100644 index 0000000000..6426aec594 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql @@ -0,0 +1,32 @@ +-- The FK columns that cascade when a workspace's apps and flows are deleted. Unindexed, +-- Postgres seq-scans the whole child table once per deleted parent row, making a workspace +-- delete cost O(apps and flows deleted x rows in the instance). Fork deletion is where that +-- bites: a fork clones its parent's apps, flows and entire app version history. +-- +-- The workspace_runnable_dependencies pair are partial because the table's check constraint +-- makes app_path and flow_path mutually exclusive, halving each index -- the cascade's +-- equality on the path proves the predicate. The table's existing path indexes are partial on +-- script_hash, which the cascade does not constrain, so they cannot serve it. +-- +-- Dropped before built: a failed concurrent build leaves an invalid index that IF NOT EXISTS +-- would accept forever, unused by the planner yet still maintained on every write. +-- +-- No statement separators outside the statements below, comments included: the CONCURRENTLY +-- rewrite in windmill-api/src/db.rs splits the file on them and would run comment text as SQL. +DROP INDEX IF EXISTS index_app_version_on_app_id; + +CREATE INDEX index_app_version_on_app_id ON app_version (app_id); + +DROP INDEX IF EXISTS index_app_script_on_app; + +CREATE INDEX index_app_script_on_app ON app_script (app); + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_app_path + ON workspace_runnable_dependencies (app_path, workspace_id) WHERE app_path IS NOT NULL; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_flow_path + ON workspace_runnable_dependencies (flow_path, workspace_id) WHERE flow_path IS NOT NULL; diff --git a/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql new file mode 100644 index 0000000000..f615c55463 --- /dev/null +++ b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql @@ -0,0 +1,6 @@ +-- Add down migration script here +-- Nothing to undo. The up migration gives a policy that only ever carried the address the +-- principal it runs as, and rewrites an address that disagreed with its principal. The previous +-- version reads both halves, so both results are correct for it too, and the addresses replaced +-- named an account other than the one the app runs as. +SELECT 1; diff --git a/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql new file mode 100644 index 0000000000..fe97d7529c --- /dev/null +++ b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql @@ -0,0 +1,100 @@ +-- Add up migration script here +-- `policy.on_behalf_of` becomes the authority for an app's identity: the address beside it is +-- written through from it on every save, so the two can no longer name different accounts. +-- +-- The address key is deliberately NOT removed here, and is still written: a replica predating +-- the derive-when-absent fallback errors outright when it is missing, which would 400 every +-- anonymous, publisher and guest app served by one that has not yet rolled over. Removing the +-- key is a follow-up, per docs/app-policy-email-removal.md. +-- +-- What is left is the data written before that rule. A policy that only ever had the address has +-- no principal to run as, so give it one. A policy whose halves disagree was stored as a client +-- sent it; reads return that pair and a redeploy that keeps the identity sends it back, where the +-- pair check rejects it. So once every policy has a principal, rewrite its address from it. + +-- Mirrors `users::username_to_permissioned_as`: an email-shaped username is its own principal +-- unless it contains a slash, which a reader would split on, and a legacy `group-*` username is +-- the group it names. +CREATE OR REPLACE FUNCTION pg_temp.username_to_permissioned_as(name VARCHAR) +RETURNS VARCHAR AS $$ + SELECT CASE + WHEN $1 LIKE '%@%' AND $1 LIKE '%/%' THEN 'u/' || $1 + WHEN $1 LIKE '%@%' THEN $1 + WHEN $1 LIKE 'group-%' THEN 'g/' || substr($1, 7) + ELSE 'u/' || $1 + END; +$$ LANGUAGE SQL IMMUTABLE; + +-- Mirrors `users::permissioned_as_from_email`: a real account wins over the synthetic group +-- namespace, which is not reserved and may be a user's own address. `pg_temp` lives for the +-- whole session and migrations share one connection, so an identically-named helper from an +-- earlier migration is still in scope: replace it, and drop this one at the end. +CREATE OR REPLACE FUNCTION pg_temp.permissioned_as_from_email(w_id VARCHAR, email VARCHAR) +RETURNS VARCHAR AS $$ + SELECT COALESCE( + (SELECT pg_temp.username_to_permissioned_as(u.username) + FROM usr u WHERE u.workspace_id = $1 AND u.email = $2), + -- A superadmin acting outside their workspaces has no usr row. + (SELECT pg_temp.username_to_permissioned_as(COALESCE(p.username, p.email)) + FROM password p WHERE p.email = $2 AND p.super_admin), + (SELECT 'g/' || g.name FROM group_ g + WHERE g.workspace_id = $1 + AND $2 = 'group-' || g.name || '@windmill.dev') + ); +$$ LANGUAGE SQL STABLE; + +-- Mirrors `users::get_email_from_permissioned_as`, except that a `u/` principal naming nobody +-- yields NULL rather than the synthetic `@unknown.windmill.dev` address, so that row is left as +-- it is instead of losing the one address it had. +CREATE OR REPLACE FUNCTION pg_temp.email_from_permissioned_as(w_id VARCHAR, principal VARCHAR) +RETURNS VARCHAR AS $$ + SELECT CASE + WHEN $2 LIKE 'u/%' THEN COALESCE( + (SELECT u.email FROM usr u WHERE u.workspace_id = $1 AND u.username = substr($2, 3)), + (SELECT p.email FROM password p + WHERE (p.username = substr($2, 3) OR p.email = substr($2, 3)) AND p.super_admin + ORDER BY p.email LIMIT 1)) + WHEN $2 LIKE 'g/%' THEN 'group-' || substr($2, 3) || '@windmill.dev' + ELSE $2 + END; +$$ LANGUAGE SQL STABLE; + +-- A policy naming only the address predates the principal being written to it. +-- A principal wider than `v2_job.permissioned_as` could not be enqueued, so it is not recorded +-- at all — the app falls back to erroring on anonymous execution until someone picks an identity +-- the deploy path accepts. Same cap and reason as the sibling migration 20260801043001. +UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], + to_jsonb(pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email'))) + WHERE policy->>'on_behalf_of' IS NULL + AND pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email') IS NOT NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email')) <= 55; + +-- App drafts carry a copy of the policy and are deployed from it, so they need the same. +UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], + to_jsonb(pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email')))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' IS NULL + AND pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email') IS NOT NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email')) <= 55; + +-- The address a save now writes, applied to the rows saved before. Execution already takes a `u/` +-- principal's own address, so this only changes what runs for a `g/` or bare principal, whose +-- stored address decided the superadmin flag and instance groups: those now follow the principal. +UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of_email'], + to_jsonb(pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of'))) + WHERE policy->>'on_behalf_of' IS NOT NULL + AND pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of') IS NOT NULL + AND policy->>'on_behalf_of_email' + IS DISTINCT FROM pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of'); + +UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], + to_jsonb(pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of')))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' IS NOT NULL + AND pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of') IS NOT NULL + AND value->'policy'->>'on_behalf_of_email' + IS DISTINCT FROM pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of'); + +DROP FUNCTION pg_temp.permissioned_as_from_email(VARCHAR, VARCHAR); +DROP FUNCTION pg_temp.email_from_permissioned_as(VARCHAR, VARCHAR); +DROP FUNCTION pg_temp.username_to_permissioned_as(VARCHAR); diff --git a/backend/migrations/20260911085230_notify_user_email_change.down.sql b/backend/migrations/20260911085230_notify_user_email_change.down.sql new file mode 100644 index 0000000000..7f6f494115 --- /dev/null +++ b/backend/migrations/20260911085230_notify_user_email_change.down.sql @@ -0,0 +1,8 @@ +-- Add down migration script here +DROP TRIGGER IF EXISTS password_superadmin_delete_trigger ON password; +DROP TRIGGER IF EXISTS password_superadmin_insert_trigger ON password; +DROP TRIGGER IF EXISTS password_superadmin_update_trigger ON password; +DROP TRIGGER IF EXISTS usr_email_update_trigger ON usr; +DROP TRIGGER IF EXISTS usr_email_change_trigger ON usr; +DROP FUNCTION IF EXISTS notify_superadmin_identity_change(); +DROP FUNCTION IF EXISTS notify_usr_email_change(); diff --git a/backend/migrations/20260911085230_notify_user_email_change.up.sql b/backend/migrations/20260911085230_notify_user_email_change.up.sql new file mode 100644 index 0000000000..f4e4a818f9 --- /dev/null +++ b/backend/migrations/20260911085230_notify_user_email_change.up.sql @@ -0,0 +1,85 @@ +-- Add up migration script here +-- Emit a notify_event so every process evicts its cached `permissioned_as` -> address mapping +-- (windmill-common EMAIL_CACHE) at its next notify-event poll, rather than serving the old +-- address for the rest of the TTL. Authorization does not rest on this: +-- `fetch_authed_from_permissioned_as` re-resolves the address from the principal's live binding. +-- SECURITY DEFINER so the INSERT runs as the function owner rather than the invoking +-- windmill_user/windmill_admin role, matching the other notify_* triggers. +CREATE OR REPLACE FUNCTION notify_usr_email_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) + VALUES ( + 'notify_user_email_change', + COALESCE(NEW.workspace_id, OLD.workspace_id) || ':' || COALESCE(NEW.username, OLD.username) + ); + -- A rename leaves the OLD username cached against this account's address; evict both keys. + IF TG_OP = 'UPDATE' AND NEW.username IS DISTINCT FROM OLD.username THEN + INSERT INTO notify_event (channel, payload) + VALUES ('notify_user_email_change', OLD.workspace_id || ':' || OLD.username); + END IF; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +-- INSERT matters too: a lookup that resolved to nobody is cached as the synthetic +-- `{username}@unknown.windmill.dev`, so creating the row has to drop that entry. +CREATE TRIGGER usr_email_change_trigger +AFTER INSERT OR DELETE ON usr +FOR EACH ROW +EXECUTE FUNCTION notify_usr_email_change(); + +CREATE TRIGGER usr_email_update_trigger +AFTER UPDATE OF email, username ON usr +FOR EACH ROW +WHEN (OLD.email IS DISTINCT FROM NEW.email OR OLD.username IS DISTINCT FROM NEW.username) +EXECUTE FUNCTION notify_usr_email_change(); + +-- A superadmin acting outside their workspaces resolves through `password` instead, and that row +-- names no workspace of its own. The `*:` payload says so: the reader drops that name's entry in +-- every workspace rather than the whole cache, which would undo the caching on an instance that +-- rewrites these rows in bulk. Confined to superadmins because they are the only accounts the +-- `usr` triggers above cannot cover. +CREATE OR REPLACE FUNCTION notify_superadmin_identity_change() +RETURNS TRIGGER AS $$ +DECLARE + names TEXT[] := '{}'; +BEGIN + -- Every alias the principal can be spelled as: `resolve_username_to_email` matches a `u/` + -- principal against `username` OR `email`, and whichever string the caller passed is the key + -- it cached under, so one account can hold a live entry under either. Old and new of each, + -- because a change to one leaves the other's entry behind. + IF TG_OP <> 'DELETE' THEN names := names || ARRAY[NEW.username, NEW.email]; END IF; + IF TG_OP <> 'INSERT' THEN names := names || ARRAY[OLD.username, OLD.email]; END IF; + INSERT INTO notify_event (channel, payload) + SELECT DISTINCT 'notify_user_email_change', '*:' || n + FROM unnest(names) AS n + WHERE n IS NOT NULL; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +-- `super_admin` is half of what the fallback matches on, so gaining or losing it moves the +-- mapping as surely as the address does: a demotion leaves the real address cached where the +-- truth is now synthetic, and a promotion leaves that synthetic one cached in place of a real +-- account. `OLD.super_admin OR NEW.super_admin` is what catches both directions. +CREATE TRIGGER password_superadmin_update_trigger +AFTER UPDATE OF email, username, super_admin ON password +FOR EACH ROW +WHEN ((OLD.super_admin OR NEW.super_admin) + AND (OLD.email IS DISTINCT FROM NEW.email + OR OLD.username IS DISTINCT FROM NEW.username + OR OLD.super_admin IS DISTINCT FROM NEW.super_admin)) +EXECUTE FUNCTION notify_superadmin_identity_change(); + +CREATE TRIGGER password_superadmin_insert_trigger +AFTER INSERT ON password +FOR EACH ROW +WHEN (NEW.super_admin) +EXECUTE FUNCTION notify_superadmin_identity_change(); + +CREATE TRIGGER password_superadmin_delete_trigger +AFTER DELETE ON password +FOR EACH ROW +WHEN (OLD.super_admin) +EXECUTE FUNCTION notify_superadmin_identity_change(); diff --git a/backend/migrations/20260909110835_grant_draft_id_seq.down.sql b/backend/migrations/20260914133552_grant_draft_id_seq.down.sql similarity index 100% rename from backend/migrations/20260909110835_grant_draft_id_seq.down.sql rename to backend/migrations/20260914133552_grant_draft_id_seq.down.sql diff --git a/backend/migrations/20260909110835_grant_draft_id_seq.up.sql b/backend/migrations/20260914133552_grant_draft_id_seq.up.sql similarity index 100% rename from backend/migrations/20260909110835_grant_draft_id_seq.up.sql rename to backend/migrations/20260914133552_grant_draft_id_seq.up.sql diff --git a/backend/migrations/20260910181644_draft_base_version.down.sql b/backend/migrations/20260914133558_draft_base_version.down.sql similarity index 100% rename from backend/migrations/20260910181644_draft_base_version.down.sql rename to backend/migrations/20260914133558_draft_base_version.down.sql diff --git a/backend/migrations/20260910181644_draft_base_version.up.sql b/backend/migrations/20260914133558_draft_base_version.up.sql similarity index 100% rename from backend/migrations/20260910181644_draft_base_version.up.sql rename to backend/migrations/20260914133558_draft_base_version.up.sql diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index cb874eb2c7..bc2498a943 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -29,6 +29,10 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/spreadsheets"], + "scope_options": [ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/spreadsheets.readonly" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -38,6 +42,11 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/drive"], + "scope_options": [ + "https://www.googleapis.com/auth/drive.file", + "https://www.googleapis.com/auth/drive.readonly", + "https://www.googleapis.com/auth/drive" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -47,6 +56,13 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/gmail.send"], + "scope_options": [ + "https://www.googleapis.com/auth/gmail.send", + "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/gmail.compose", + "https://www.googleapis.com/auth/gmail.modify", + "https://www.googleapis.com/auth/gmail.labels" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -56,6 +72,12 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/calendar.events"], + "scope_options": [ + "https://www.googleapis.com/auth/calendar.events", + "https://www.googleapis.com/auth/calendar.events.readonly", + "https://www.googleapis.com/auth/calendar.readonly", + "https://www.googleapis.com/auth/calendar" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -65,6 +87,12 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/forms"], + "scope_options": [ + "https://www.googleapis.com/auth/forms", + "https://www.googleapis.com/auth/forms.body", + "https://www.googleapis.com/auth/forms.body.readonly", + "https://www.googleapis.com/auth/forms.responses.readonly" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -74,6 +102,10 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/cloud-platform"], + "scope_options": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -88,6 +120,15 @@ "https://www.googleapis.com/auth/admin.directory.user.security", "https://www.googleapis.com/auth/admin.directory.orgunit" ], + "scope_options": [ + "https://www.googleapis.com/auth/admin.directory.user", + "https://www.googleapis.com/auth/admin.directory.user.readonly", + "https://www.googleapis.com/auth/admin.directory.group", + "https://www.googleapis.com/auth/admin.directory.group.readonly", + "https://www.googleapis.com/auth/admin.directory.orgunit", + "https://www.googleapis.com/auth/admin.directory.orgunit.readonly", + "https://www.googleapis.com/auth/admin.directory.user.security" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -200,6 +241,7 @@ } }, "snowflake_oauth": { + "resource_fields": ["database", "warehouse", "role", "schema"], "connect_config_template": { "display_name": "Snowflake", "label": "Snowflake Account Identifier", diff --git a/backend/oauth_login.json b/backend/oauth_login.json index 93cdaac2b2..4706429404 100644 --- a/backend/oauth_login.json +++ b/backend/oauth_login.json @@ -15,13 +15,19 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "userinfo_url": "https://www.googleapis.com/oauth2/v1/userinfo?alt=json", - "scopes": ["https://www.googleapis.com/auth/userinfo.email"] + "scopes": ["https://www.googleapis.com/auth/userinfo.email"], + "extra_params": { + "prompt": "select_account" + } }, "microsoft": { "auth_url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", "token_url": "https://login.microsoftonline.com/common/oauth2/v2.0/token", "userinfo_url": "https://graph.microsoft.com/oidc/userinfo", - "scopes": ["openid", "profile", "email"] + "scopes": ["openid", "profile", "email"], + "extra_params": { + "prompt": "select_account" + } }, "jumpcloud": { "auth_url": "https://oauth.id.jumpcloud.com/oauth2/auth", diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index b0ed12fb1a..271ad4835b 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -694,6 +694,107 @@ pub fn remove_pinned_imports(code: &str) -> anyhow::Result { Ok(content) } +/// Spans of the string literals naming a loaded module: `import`/`export … from` sources and the +/// argument of a dynamic `import()`. A `require()` call is left out: `require` is an ordinary +/// binding a script can shadow, so its argument is not known to be a module. +struct ImportSpecifierSpans(Vec); + +impl Visit for ImportSpecifierSpans { + noop_visit_type!(); + + fn visit_import_decl(&mut self, n: &swc_ecma_ast::ImportDecl) { + self.0.push(n.src.span); + } + + fn visit_export_all(&mut self, n: &swc_ecma_ast::ExportAll) { + self.0.push(n.src.span); + } + + fn visit_named_export(&mut self, n: &swc_ecma_ast::NamedExport) { + if let Some(src) = &n.src { + self.0.push(src.span); + } + } + + fn visit_call_expr(&mut self, n: &swc_ecma_ast::CallExpr) { + if let (swc_ecma_ast::Callee::Import(_), Some(arg)) = (&n.callee, n.args.first()) { + if let (None, Expr::Lit(Lit::Str(s))) = (arg.spread, &*arg.expr) { + self.0.push(s.span); + } + } + n.visit_children_with(self); + } +} + +/// Drops the `@version` from each pinned module specifier (`pkg@1.2.3/sub` -> `pkg/sub`), +/// rewriting only the specifier literals. Unlike [`remove_pinned_imports`], the same text +/// elsewhere, such as a string the script returns, stays as written. +pub fn remove_pinned_import_specifiers(code: &str) -> anyhow::Result { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file( + FileName::Custom("main.d.ts".into()).into(), + code.to_string(), + ); + let mut tss = TsSyntax::default(); + tss.tsx = true; + tss.no_early_errors = true; + let lexer = Lexer::new( + Syntax::Typescript(tss), + Default::default(), + StringInput::from(&*fm), + None, + ); + let module = Parser::new_from(lexer).parse_module().map_err(|e| { + anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {e:?}") + })?; + let mut specifiers = ImportSpecifierSpans(vec![]); + specifiers.visit_module(&module); + specifiers.0.sort_by_key(|s| s.lo); + + // Spans index the parsed source, which the source map stripped of any UTF-8 BOM. + let bom = if code.starts_with('\u{feff}') { + '\u{feff}'.len_utf8() + } else { + 0 + }; + let offset = + |pos: swc_common::BytePos| pos.0.checked_sub(fm.start_pos.0).map(|o| bom + o as usize); + let mut content = String::with_capacity(code.len()); + let mut copied = 0; + for span in specifiers.0 { + // A span covers the literal's quotes. One that does not land on a matching pair is left + // as written rather than risk rewriting the wrong bytes. + let (Some(open), Some(close)) = ( + offset(span.lo), + offset(span.hi).and_then(|e| e.checked_sub(1)), + ) else { + continue; + }; + let quote = code.as_bytes().get(open); + if open >= close + || open < copied + || !matches!(quote, Some(b'"' | b'\'')) + || code.as_bytes().get(close) != quote + { + continue; + } + let Some(specifier) = code.get(open + 1..close) else { + continue; + }; + let unpinned = IMPORTS_VERSION.captures(specifier).and_then(|x| { + x.get(1) + .map(|y| format!("{}{}", y.as_str(), x.get(2).map_or("", |z| z.as_str()))) + }); + if let Some(unpinned) = unpinned.filter(|u| u != specifier) { + content.push_str(&code[copied..open + 1]); + content.push_str(&unpinned); + copied = close; + } + } + content.push_str(&code[copied..]); + Ok(content) +} + fn resolve_type_ref(type_resolver: &HashMap, typ: &mut Typ) { let mut visited = std::collections::HashSet::new(); resolve_type_ref_with_visited(type_resolver, typ, &mut visited); diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 4309018fb4..4206dd68f8 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -4,6 +4,7 @@ mod tests { use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; use windmill_parser_ts::{ parse_deno_signature, parse_expr_for_imports, parse_relative_imports, + remove_pinned_import_specifiers, }; #[test] @@ -33,6 +34,46 @@ mod tests { ); } + #[test] + fn test_remove_pinned_import_specifiers_rewrites_only_specifiers() { + let code = r#"// héllo +import a from "pkg@1.2.3"; +import b from "@scope/pkg@^2/sub"; +export * from "other@3"; +import rel from "./helper"; +const c = await import("dyn@4"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"#; + assert_eq!( + remove_pinned_import_specifiers(code).unwrap(), + r#"// héllo +import a from "pkg"; +import b from "@scope/pkg/sub"; +export * from "other"; +import rel from "./helper"; +const c = await import("dyn"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"# + ); + assert_eq!( + remove_pinned_import_specifiers("\u{feff}import a from 'pkg@1';").unwrap(), + "\u{feff}import a from 'pkg';" + ); + assert_eq!( + remove_pinned_import_specifiers( + "// a\r\n// b\r\nimport a from \"pkg@1\";\r\nimport b from 'x@2';" + ) + .unwrap(), + "// a\r\n// b\r\nimport a from \"pkg\";\r\nimport b from 'x';" + ); + } + #[test] fn test_parse_empty_main_signature() { let code = r#" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 395e692623..bf2f75120e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.803.0" +version = "1.811.1" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.803.0" +version = "1.811.1" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.803.0" +version = "1.811.1" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.803.0" +version = "1.811.1" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 45c8b0d94e..e7ee625be9 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.803.0" +version = "1.811.1" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs index 24f1e2c39d..36617c5639 100644 --- a/backend/parsers/windmill-parser-yaml/src/dbt.rs +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -58,6 +58,31 @@ impl DbtEngine { pub fn emits_node_events(&self) -> bool { matches!(self, DbtEngine::DbtCore1x) } + + /// Whether the engine's CLI has `--write-index`, the flag that writes the + /// parquet index column lineage lives in. False for 1.x, whose Python CLI + /// has no such option. + /// + /// True is not a promise that the artifact appears: `dbt-core` 2.0.0-alpha.5 + /// accepts the flag, declares the views over `dbt.column_lineage` in its own + /// `views.sql`, and writes neither that parquet nor `dbt.node_columns`. Only + /// Fusion does today. Attempting the pass on both is what lets a later 2.x + /// release pick the feature up with no change here. + pub fn writes_column_index(&self) -> bool { + !matches!(self, DbtEngine::DbtCore1x) + } + + /// Whether the engine has `--defer-state`, the deferral-only half of + /// `--state`. + /// + /// It matters on one command. `dbt retry` reads the run it resumes from + /// `--state`, so an engine with only that flag cannot be told to defer and + /// to resume from the job's own results at once: handed the deferral's + /// directory, it resumes the all-green run stored there and rebuilds + /// nothing. Only dbt-core 1.x separates the two. + pub fn has_defer_state_flag(&self) -> bool { + matches!(self, DbtEngine::DbtCore1x) + } } /// How the warehouse connection is supplied. Both paths are supported @@ -127,6 +152,18 @@ pub struct DbtDescriptor { pub selector: Option, #[serde(default)] pub test_behavior: DbtTestBehavior, + /// Ingest column-to-column lineage and the real column schemas, from the + /// engine's static analysis. + /// + /// Opt-in, and it has to be: the artifact only appears under + /// `--static-analysis strict`, which rejects SQL the default accepts (an + /// unresolvable identifier is an error there and compiles fine otherwise). + /// Turning it on for everyone would make a stricter dialect the price of + /// deploying a dbt project. It is a separate `dbt compile` pass, so nothing + /// it decides can change what a build does; a project it cannot analyze + /// keeps the graph it has today. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub column_lineage: bool, /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are /// all normal — so values keep their YAML type; only string leaves carry /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing @@ -137,6 +174,16 @@ pub struct DbtDescriptor { pub threads: Option, #[serde(default)] pub full_refresh: bool, + /// Resolve a `ref()` a run does not build through the state the last + /// successful run of this environment published, rather than through the + /// schema that run writes into. + /// + /// Only the default for the `build` block's own `defer`, since the choice is + /// per run: the run that publishes an environment's state and the run that + /// defers to it are two invocations of ONE script (decision 6), so a project + /// that could only defer by descriptor could never populate what it reads. + #[serde(default)] + pub defer: bool, /// Automatic in-job retry of the nodes a build failed on. /// /// dbt already confines a failure to its own subtree, and `dbt retry` @@ -258,6 +305,7 @@ pub const RESERVED_ARG_NAMES: &[&str] = &[ "exclude", "vars", "full_refresh", + "defer", "dbt_command", "dbt_retry_job", "model", @@ -345,15 +393,26 @@ fn command_variants(d: &DbtDescriptor) -> Vec<(&'static str, Vec)> { "build", selection() .into_iter() - .chain([Arg { - name: "full_refresh".to_string(), - otyp: None, - typ: Typ::Bool, - has_default: true, - default: Some(serde_json::json!(d.full_refresh)), - oidx: None, - otyp_inferred: false, - }]) + .chain([ + Arg { + name: "full_refresh".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.full_refresh)), + oidx: None, + otyp_inferred: false, + }, + Arg { + name: "defer".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.defer)), + oidx: None, + otyp_inferred: false, + }, + ]) .collect(), ), ( @@ -546,7 +605,9 @@ fn property_of(arg: &Arg) -> serde_json::Value { ), "select" => Some( "dbt selection syntax, e.g. `tag:nightly`, `stg_orders+`, \ - `config.materialized:incremental`. Empty runs the descriptor's own selection.", + `config.materialized:incremental`. `state:modified+` and `result:error+` \ + compare against the state a previous run published, so they need `defer` on. \ + Empty runs the descriptor's own selection.", ), "exclude" => Some("Nodes to leave out of the selection above, same syntax."), "vars" => Some( @@ -554,6 +615,11 @@ fn property_of(arg: &Arg) -> serde_json::Value { exist makes this run store its own graph rather than the deployed one.", ), "full_refresh" => Some("Rebuild incremental models from scratch instead of appending."), + "defer" => Some( + "Resolve a `ref()` this run does not build to the relation the last successful \ + run of this warehouse and target published, instead of to the schema this run \ + writes into.", + ), "model" => Some( "The model to preview, by name — `stg_orders`, or `my_package.stg_orders` when \ two packages share a name. Any dbt selector resolving to ONE node works.", @@ -687,8 +753,15 @@ full_refresh: true }; let (build, build_args) = of("build"); - assert_eq!(build_args, ["exclude", "full_refresh", "select", "vars"]); + assert_eq!( + build_args, + ["defer", "exclude", "full_refresh", "select", "vars"] + ); assert_eq!(build["properties"]["full_refresh"]["type"], "boolean"); + // `defer` is a per-run toggle rather than a descriptor-only setting: the + // run that publishes an environment's state and the run that defers to + // it are two invocations of ONE script. + assert_eq!(build["properties"]["defer"]["type"], "boolean"); // Defaults come from the descriptor, so an untouched run reproduces it. assert_eq!( build["properties"]["select"]["default"], diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 843a83a310..30c6047255 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -29,10 +29,10 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, the warehouse named as the - /// workspace configures it. The scheme names the producer, the path stays - /// the relation — see `windmill_types::AssetKind::Dbt`. + /// A warehouse relation, `dbt:////`, the warehouse + /// named as the workspace configures it. The scheme names the namespace dbt + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } @@ -288,11 +288,13 @@ pub struct RetrySpec { } // `// materialize [manual] [append] [key=] [history] [track=]` -// — declares that this script produces a *managed* materialization of `` -// (a `ducklake://` table). By default the runtime generates the write DDL around +// — declares that this script produces ``. A `ducklake://` table is +// materialized *managed* by default: the runtime generates the write DDL around // the script's single trailing `SELECT` and owns idempotency, partition-state // and snapshot capture. `manual` is the escape hatch: the script writes its own -// DDL and the runtime only records state (track-only). The reconciliation +// DDL and the runtime only records state (track-only) — and it is the only mode a +// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy +// enforces that; see docs/dbt-runtime.md). The reconciliation // strategy options apply to managed mode: none → DELETE-by-partition + INSERT // (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). @@ -804,6 +806,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String { ) } +/// Whether a `dbt://` path names a whole relation, `//`. +/// +/// Every producer spells one that way — the manifest ingest derives it from +/// `relation_name`, a `// materialize` target is checked against it — so anything +/// else can be produced by nothing and read by nothing. Both sides of the deploy +/// ask here rather than counting segments themselves: a subscription and a write +/// that disagreed on the shape would refuse and accept the same string. +pub fn is_full_relation_path(path: &str) -> bool { + let mut segments = path.split('/'); + segments.clone().count() == 3 && !segments.any(str::is_empty) +} + /// A doubled delimiter inside a quoted identifier is that delimiter, literally — /// the same rule the worker's `split_relation` applies to `relation_name`. Both /// have to decode it or one spelling of a table becomes two graph nodes: the dbt @@ -1740,10 +1754,7 @@ mod pipeline_annotation_tests { // just stop being the same node and the cross-boundary cascade never fires. #[test] fn table_paths_from_every_spelling_canonicalize_to_one_key() { - let canonical = Some(( - AssetKind::Dbt, - Cow::Owned("main/analytics/orders".into()), - )); + let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into()))); for spelling in [ // Hand-written annotation. "dbt://main/analytics/orders", @@ -1765,6 +1776,17 @@ mod pipeline_annotation_tests { } } + /// The shape both halves of the deploy check against: a subscription and a + /// write that disagreed on it would refuse and accept the same string. + #[test] + fn a_whole_relation_is_three_non_empty_segments() { + assert!(is_full_relation_path("main/analytics/orders")); + assert!(is_full_relation_path("main/archive.sales/orders")); + for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] { + assert!(!is_full_relation_path(partial), "{partial} is not a relation"); + } + } + // A relation that overrode its database carries `.` in // one segment, and each half can be quoted independently. Stripping only // the outer pair leaves a key the manifest ingest never produces, so the @@ -1791,10 +1813,7 @@ mod pipeline_annotation_tests { // database qualifier. assert_eq!( parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), - Some(( - AssetKind::Dbt, - Cow::Owned("main/sales.v2/orders".into()) - )) + Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into()))) ); } @@ -1813,14 +1832,8 @@ mod pipeline_annotation_tests { "dbt://main/analytics/\"order\"\"s\"", "main/analytics/order\"s", ), - ( - "dbt://main/`da``ta`/`orders`", - "main/da`ta/orders", - ), - ( - "dbt://main/[my]]schema]/[orders]", - "main/my]schema/orders", - ), + ("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"), + ("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"), // And in one half of a database-qualified segment. ( "dbt://main/\"arch\"\"ive\".\"sales\"/orders", @@ -1839,8 +1852,14 @@ mod pipeline_annotation_tests { // apart. A lone delimiter treated as opening a quote would be dropped — // `sa"les` filed as `sales` — and the two derivations would split. for (decoded, spelled) in [ - ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), - ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/sa\"les/orders", + "dbt://main/\"sa\"\"les\"/orders", + ), + ( + "dbt://main/analytics/order\"s", + "dbt://main/analytics/\"order\"\"s\"", + ), ( "dbt://main/arch\"ive.sales/orders", "dbt://main/\"arch\"\"ive\".\"sales\"/orders", diff --git a/backend/src/main.rs b/backend/src/main.rs index f84812151d..7f2ce3cd82 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -46,7 +46,8 @@ use windmill_common::{ CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, @@ -135,7 +136,8 @@ use crate::monitor::{ reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting, reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, - reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting, + reload_extra_pip_index_url_setting, reload_http_route_default_allowed_origins_setting, + reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, @@ -1913,6 +1915,17 @@ async fn process_notify_event( ); windmill_api::auth::invalidate_token_from_cache(payload); } + "notify_user_email_change" => { + // `:`, or `*:` from a `password` change, which + // knows the name but no workspace. Workspace ids can't contain ':'. + if let Some(username) = payload.strip_prefix("*:") { + tracing::info!("Superadmin identity change detected, invalidating: {username}"); + windmill_common::users::invalidate_email_cache_for_username(username); + } else if let Some((workspace_id, username)) = payload.split_once(':') { + tracing::info!("User email change detected, invalidating cache: {payload}"); + windmill_common::users::invalidate_email_cache(workspace_id, username); + } + } "notify_app_policy_change" => { // payload is `:`; workspace ids can't contain ':'. if server_mode { @@ -1944,6 +1957,13 @@ async fn process_notify_event( if let Err(e) = reload_license_key(&db.into()).await { tracing::error!("Failed to reload license key: {e:#}"); } + // The worker-group cache override is Enterprise-only, and nothing else + // re-reads the plan for it: the periodic settings pass runs ahead of + // reload_license_key, so it would see the plan this event just replaced. + #[cfg(feature = "parquet")] + if worker_mode { + crate::monitor::reload_cache_object_store_override_with_retry(db).await; + } } DEFAULT_TAGS_PER_WORKSPACE_SETTING => { if let Err(e) = load_tag_per_workspace_enabled(db).await { @@ -2127,6 +2147,11 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload app workspaced route setting"); } } + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING => { + if let Err(e) = reload_http_route_default_allowed_origins_setting(db).await { + tracing::error!(error = %e, "Could not reload http route default allowed origins setting"); + } + } HTTP_ROUTE_WORKSPACED_ROUTE_SETTING => { if let Err(e) = reload_http_route_workspaced_route_setting(db).await { tracing::error!(error = %e, "Could not reload http route workspaced route setting"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 86e75b7de9..dc8072d684 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -106,8 +106,13 @@ use windmill_common::{ use windmill_common::{ client::AuthedClient, global_settings::{ - APP_WORKSPACED_ROUTE_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE, - HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + parse_allowed_origins_setting, APP_WORKSPACED_ROUTE_SETTING, + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + HTTP_ROUTE_WORKSPACED_ROUTE, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + }, + queue_metrics::{ + QueueSample, QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX, QUEUE_DELAY_SAME_HEAD_SECS, + QUEUE_METRIC_HEARTBEAT_SECS, QUEUE_METRIC_STALE_SECS, }, }; #[cfg(feature = "parquet")] @@ -354,7 +359,9 @@ pub async fn initial_load( ) } }); - pass.action(windmill_common::min_version::store_min_keep_alive_version(db)); + pass.action(windmill_common::min_version::store_min_keep_alive_version( + db, + )); pass.setting( windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING, false, @@ -396,6 +403,8 @@ pub async fn initial_load( additional_python_paths: None, pip_local_dependencies: None, native_mode, + // an agent worker never reads its group's config, only its token + object_store_cache_config: None, })); } } @@ -420,6 +429,18 @@ pub async fn initial_load( pass.setting(APP_WORKSPACED_ROUTE_SETTING, false, |v| async move { apply_app_workspaced_route_setting(v) }); + pass.setting( + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + false, + |v| async move { + if let Err(e) = apply_http_route_default_allowed_origins_setting(v) { + tracing::error!( + "Error reloading http route default allowed origins: {:?}", + e + ) + } + }, + ); pass.setting( HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, false, @@ -709,7 +730,6 @@ pub async fn initial_load( pass.run(conn).await; } - pub fn apply_metrics_enabled(value: Option) { if let Some(serde_json::Value::Bool(t)) = value { METRICS_ENABLED.store(t, Ordering::Relaxed) @@ -1066,8 +1086,8 @@ pub fn apply_fork_workspace_tag_append_fork_suffix(value: Option error::Result<()> { - let v = - load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await?; + let v = load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true) + .await?; apply_critical_alert_mute_ui_setting(v); Ok(()) } @@ -1937,6 +1957,15 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error deleting old feature_usage rows: {e}"); } + // Guest sign-ins, kept a month longer than the seat window they feed so a late + // telemetry send still sees a whole month. + if let Err(e) = sqlx::query!("DELETE FROM guest_activity WHERE day < CURRENT_DATE - 60") + .execute(db) + .await + { + tracing::error!("Error deleting old guest_activity rows: {e}"); + } + match sqlx::query_scalar!( "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", ) @@ -2723,7 +2752,6 @@ pub async fn reload_timeout_wait_result_setting(conn: &Connection) { .await; } - pub async fn reload_extra_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2814,7 +2842,6 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) { .await; } - pub async fn reload_nuget_config_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2922,7 +2949,6 @@ pub async fn reload_ruby_repos_setting(conn: &Connection) { .await; } - pub async fn reload_workspace_registries_setting(conn: &Connection) { match load_value_from_global_settings_with_conn( conn, @@ -3176,7 +3202,6 @@ pub async fn apply_job_isolation_setting(value: Option) { } } - async fn resolve_license_key_value(conn: &Connection, quiet: bool) -> anyhow::Result { let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true) .await @@ -3471,7 +3496,10 @@ impl<'a> SettingsPass<'a> { // on compile-time defaults until the next full reload. Only the single-query transport // can fail this way; over HTTP the batch already is the per-setting read. if matches!(conn, Connection::Sql(_)) && values.is_empty() && !names.is_empty() { - tracing::warn!("Falling back to per-setting reads for {} settings", names.len()); + tracing::warn!( + "Falling back to per-setting reads for {} settings", + names.len() + ); values = fetch_settings_individually(conn, &names).await; } for (name, http) in &declared { @@ -3863,7 +3891,6 @@ pub fn parse_setting_value( value } - #[cfg(feature = "prometheus")] pub async fn monitor_pool(db: &DB) { if METRICS_ENABLED.load(Ordering::Relaxed) { @@ -4307,6 +4334,24 @@ pub async fn monitor_db( } }; + // Re-check what each git-sync repository's own credential says about its expiry, + // and rotate the ones close to it. Every ~40 min: the values move over days, and + // `should_run` counts iterations in a u8. Spawned rather than joined: the join + // below is cancelled at its deadline, which a long sweep would reach, and a + // rotation cut off between GitLab issuing a token and Windmill storing it + // loses the token family. Detached, only process shutdown can cut it off, + // which a rotation almost never coincides with. The pass's advisory lock + // keeps a slow one from overlapping the next. + let git_credential_maintenance_f = async { + #[cfg(all(feature = "enterprise", feature = "private"))] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { maintain_git_credentials(&db).await }); + } + } + }; + // run every 2 iterations (~20s at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). // Enterprise feature: the active `// freshness` backstop lives in // windmill-queue's `freshness_watchdog` (`private`); OSS gets a no-op stub. @@ -4360,6 +4405,7 @@ pub async fn monitor_db( export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, git_auto_pull_f, + git_credential_maintenance_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, ); @@ -4665,6 +4711,14 @@ pub async fn poll_git_auto_pull(db: &Pool) { { tracing::error!("git auto-pull: advisory unlock failed: {e:#}"); } + + // Backstop for the "Windmill CI tests" checks: retry a failed GitHub create or + // delivery, conclude checks whose tests settled, time out stuck ones, prune old + // rows. Detached and outside the advisory lock: its writes are guarded (claimed + // conclude, greatest-id upsert), it is single-flight, and its GitHub calls must not + // count against the monitor pass's budget. + let db = db.clone(); + tokio::spawn(async move { windmill_git_sync::sweep_ci_test_checks(&db).await }); } #[cfg(feature = "private")] @@ -4682,6 +4736,156 @@ lazy_static::lazy_static! { #[cfg(feature = "private")] const AUTO_PULL_POLL_SLACK_S: i64 = 30; +/// Advisory lock id ensuring only one server replica maintains git credentials at +/// a time (adjacent to GIT_AUTO_PULL_LOCK_ID). +#[cfg(all(feature = "enterprise", feature = "private"))] +const GIT_CREDENTIAL_LOCK_ID: i64 = 737_483_923; + +/// Refresh every git-sync repository's credential status and rotate the ones near +/// expiry, so a token dies visibly (and usually not at all) rather than taking +/// sync down on its expiry date. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials(db: &Pool) { + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + return; + } + + // Transaction-scoped advisory lock, as for the schedule reconcile above: a + // session lock on a pooled connection would ride back into the pool still + // held if the sweep died before unlocking, and wedge the pass on every + // replica until a restart. The transaction only owns the lock; the sweep + // commits each status and each rotated token on its own as it goes. + let mut lock_tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("git credentials: failed to begin lock tx: {e:#}"); + return; + } + }; + // The transaction stays idle while the sweep talks to git hosts, and the + // pool's ten-minute idle-in-transaction timeout would end it, lock included, + // partway through a sweep over enough slow hosts. Lifted for this + // transaction only; it dies with the connection either way. + if let Err(e) = sqlx::query("SET LOCAL idle_in_transaction_session_timeout = 0") + .execute(&mut *lock_tx) + .await + { + tracing::error!("git credentials: failed to lift the idle timeout: {e:#}"); + return; + } + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(GIT_CREDENTIAL_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("git credentials: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + return; + } + + if let Err(e) = maintain_git_credentials_inner(db).await { + tracing::error!("git credentials: maintenance error: {e:#}"); + } + drop(lock_tx); +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials_inner(db: &Pool) -> error::Result<()> { + use windmill_common::workspaces::WorkspaceGitSyncSettings; + + // Same deleted/archived exclusion as the auto-pull poller: a dead workspace's + // settings row survives, and rotating a token for one would be pure damage. + let rows = sqlx::query!( + r#"SELECT ws.workspace_id, ws.git_sync + FROM workspace_settings ws + JOIN workspace w ON w.id = ws.workspace_id + WHERE NOT w.deleted + AND ws.git_sync IS NOT NULL + AND jsonb_typeof(ws.git_sync->'repositories') = 'array'"# + ) + .fetch_all(db) + .await?; + + for row in rows { + let Some(git_sync) = row.git_sync else { + continue; + }; + let settings: WorkspaceGitSyncSettings = match serde_json::from_value(git_sync) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + "git credentials: invalid git_sync settings for workspace {}: {e}", + row.workspace_id + ); + continue; + } + }; + + for repo in settings.repositories.iter() { + let path = &repo.git_repo_resource_path; + // This refreshes and records the status on every repository it looks at, + // rotating only the ones near expiry, so it is the whole maintenance pass + // rather than just the rotation half. + if let Err(e) = windmill_common::git_sync_ee::rotate_git_credential_if_due( + db, + &row.workspace_id, + path, + ) + .await + { + tracing::error!( + "git credentials: maintenance failed for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + + // A repository that wants webhook delivery but holds no hook never + // gets one otherwise: the reconcile runs on a settings save, so a + // credential that was unusable when the hook should have been created + // would leave it missing until an admin saved again. Checking stored + // state costs nothing, and only the repositories actually missing a + // hook reach the host. + use windmill_common::workspaces::AutoPullMode; + // Also when a hook exists but carries a warning: a save during a GitLab + // outage keeps the hook and records why it could not be confirmed, and + // that warning is only cleared by a reconcile that confirms it again. + // Only repositories with a checked credential of their own: for + // everything else a settings save stays the one place hooks are + // reconciled, so an App repository or a plain remote is never touched + // here, and a recorded delivery mode nobody saved is never normalized. + let needs_hook = repo.credential.is_some() + && repo.auto_pull.as_ref().is_some_and(|a| { + a.enabled + && matches!(a.mode, AutoPullMode::Auto | AutoPullMode::Webhook) + && (a.webhook_id.is_none() || a.webhook_error.is_some()) + }); + if needs_hook { + let mut repo = repo.clone(); + if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook( + db, + &row.workspace_id, + &mut repo, + ) + .await + { + tracing::warn!( + "git credentials: could not reconcile the webhook for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + } + } + } + Ok(()) +} + #[cfg(feature = "private")] async fn poll_git_auto_pull_inner(db: &Pool) -> error::Result<()> { use windmill_common::workspaces::{AutoPullMode, WorkspaceGitSyncSettings}; @@ -4927,155 +5131,303 @@ async fn vacuuming_tables(db: &Pool) -> error::Result<()> { Ok(()) } -pub async fn expose_queue_metrics(db: &Pool) { - let last_check = sqlx::query_scalar!( - "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1" - ) - .fetch_optional(db) - .await - .unwrap_or(Some(chrono::Utc::now())); +/// Shortest spacing between two stored samples of the same queue metric, so a tag whose +/// value moves on every monitor round still writes at most one row per interval. Also how +/// often each server samples the queue when no Prometheus or OTel gauge needs it sooner. +const QUEUE_METRIC_MIN_INTERVAL_SECS: f64 = 25.0; +/// A held delay hovers while the head keeps changing, so an exact-value comparison would rarely +/// dedup it. Only a move the chart would actually render is stored. +const QUEUE_DELAY_TOLERANCE: f64 = 0.1; - let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); - let save_metrics = last_check - .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25)) - .unwrap_or(true); +/// Append the queue metrics the drawer at `GET /workers/queue_metrics_series` charts, skipping +/// any sample that repeats what is already stored. +/// +/// Only tags with a backlog appear in `queue_stats`, and an arbitrary `?tag=` nobody serves +/// stays backlogged forever, so writing every round would repeat the same pair of rows for +/// the whole 14-day retention. Each metric is written when the value it draws moves, once per +/// heartbeat while it holds, and once more (as a zero) when the tag drains. Gaps therefore +/// mean "unchanged since the last row", which is what the chart interpolates. A delay whose +/// head job stays put is stored as that job's wait start, which the chart draws climbing, so it +/// never moves away from what is stored either. +async fn save_queue_metrics( + db: &Pool, + queue_stats: &std::collections::HashMap, +) { + let sampled_ids = queue_stats + .keys() + .flat_map(|tag| { + [ + format!("{QUEUE_COUNT_PREFIX}{tag}"), + format!("{QUEUE_DELAY_PREFIX}{tag}"), + ] + }) + .collect::>(); - if metrics_enabled || save_metrics || OTEL_METRICS_ENABLED.load(Ordering::Relaxed) { - let queue_counts = windmill_common::queue::get_queue_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - (*QUEUE_COUNT).with_label_values(&[q]).set(0); - } - } + // Last stored sample of every metric that either has a backlog now or was written + // recently enough to still be believed backlogged. Bounding the lookup by the stale window + // keeps it cheap at any `metrics` size; a per-id `ORDER BY created_at DESC LIMIT 1` does + // not, since the planner may serve it from `metrics_sort_idx` and walk the whole table. + let last_samples = match sqlx::query!( + "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\", + EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\", + EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\" + FROM unnest($1::text[]) AS c(id) + FULL JOIN ( + SELECT DISTINCT ON (id) id, value, created_at + FROM metrics + WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2) + ORDER BY id, created_at DESC + ) r ON r.id = c.id", + &sampled_ids[..], + QUEUE_METRIC_STALE_SECS, + ) + .fetch_all(db) + .await + { + Ok(rows) => rows, + Err(e) => { + tracing::error!("Failed to read last queue metrics samples: {e:#}"); + return; } + }; - let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let mut ids = vec![]; + let mut values = vec![]; + // The wait start of the head of each held delay, whose value the INSERT computes. + let mut held_heads: Vec> = vec![]; + for row in last_samples { + let Some((prefix, tag)) = [QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX] + .into_iter() + .find_map(|p| row.id.strip_prefix(p).map(|tag| (p, tag))) + else { + continue; + }; + // A stored value that cannot be read cannot be compared, so the next reading is kept. + let last = row + .value + .as_ref() + .and_then(QueueSample::parse) + .zip(row.at) + .zip(row.age) + .map(|((sample, at), age)| (sample, at, age)); + let stat = queue_stats.get(tag); + let current = stat.map(|stat| { + if prefix == QUEUE_COUNT_PREFIX { + stat.count as f64 + } else { + stat.delay + } + }); - if otel_enabled { - for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - otel_set_queue_count(q, 0); + let next_delay = stat + .filter(|_| prefix == QUEUE_DELAY_PREFIX) + .map(|stat| delay_sample(last.map(|(sample, at, _)| sample.head_since(at)), stat)); + let drawn_now = last.map(|(sample, at, age)| (sample.value_at(at + age), age)); + let redraws = last + .zip(next_delay) + .is_some_and(|((sample, ..), next)| redraws(sample, next)); + if should_store(prefix, drawn_now, current, redraws) { + let (value, held_head) = match (stat, next_delay) { + (None, _) => (serde_json::json!(0), None), + (Some(stat), None) => (serde_json::json!(stat.count), None), + (Some(stat), Some(QueueSample::Held(_))) => { + (serde_json::Value::Null, Some(stat.head_since)) } - } - } - - #[allow(unused_mut)] - let mut tags_to_watch = vec![]; - #[allow(unused_mut)] - let mut otel_tags_to_watch = vec![]; - for q in queue_counts { - let count = q.1; - let tag = q.0; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); - metric.set(count as i64); - tags_to_watch.push(tag.to_string()); - } - - if otel_enabled { - otel_tags_to_watch.push(tag.to_string()); - } - otel_set_queue_count(&tag, count as i64); - - // save queue_count and delay metrics per tag - if save_metrics { - sqlx::query!( - "INSERT INTO metrics (id, value) VALUES ($1, $2)", - format!("queue_count_{}", tag), - serde_json::json!(count) - ) - .execute(db) - .await - .ok(); - if count > 0 { - sqlx::query!( - "INSERT INTO metrics (id, value) - VALUES ($1, to_jsonb(( - SELECT EXTRACT(EPOCH FROM now() - scheduled_for) - FROM v2_job_queue - WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval - ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1 - )))", - format!("queue_delay_{}", tag), - tag - ) - .execute(db) - .await - .ok(); - } - } - } - if metrics_enabled { - let mut w = QUEUE_COUNT_TAGS.write().await; - *w = tags_to_watch; - } - if otel_enabled { - let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; - *w = otel_tags_to_watch; - } - - // Single DB query for running counts, shared by Prometheus and OTel - let otel_running = otel_enabled; - #[cfg(feature = "prometheus")] - let need_running_counts = metrics_enabled || otel_running; - #[cfg(not(feature = "prometheus"))] - let need_running_counts = otel_running; - - if need_running_counts { - let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); - } - } - } - - if otel_running { - for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - otel_set_queue_running_count(q, 0); - } - } - } - - #[allow(unused_mut, unused_variables)] - let mut running_tags_to_watch: Vec = vec![]; - #[allow(unused_mut, unused_variables)] - let mut otel_running_tags_to_watch: Vec = vec![]; - for (tag, count) in &queue_running_counts { - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); - metric.set(*count as i64); - running_tags_to_watch.push(tag.to_string()); - } - - if otel_running { - otel_set_queue_running_count(tag, *count as i64); - otel_running_tags_to_watch.push(tag.to_string()); - } - } - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = running_tags_to_watch; - } - if otel_running { - let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = otel_running_tags_to_watch; - } + (Some(_), Some(climbing)) => (climbing.to_json(), None), + }; + ids.push(row.id); + values.push(value); + held_heads.push(held_head); } } + if ids.is_empty() { + return; + } + // A held delay is computed from this statement's `now()`, the row's `created_at` too, so + // `created_at - value` is exactly its head's wait start. That is how the next sample tells + // whether the same job is still at the head, within `QUEUE_DELAY_SAME_HEAD_SECS`, which the + // time between reading the queue and this INSERT could otherwise exceed on a busy database. + if let Err(e) = sqlx::query!( + "INSERT INTO metrics (id, value) + SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value) + FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + &ids[..], + &values[..], + &held_heads[..] as &[Option], + ) + .execute(db) + .await + { + tracing::error!("Failed to save queue metrics: {e:#}"); + } +} + +/// What to store for a delay reading, given when the head job of the last stored sample started +/// waiting. The same job still at the head keeps the delay climbing from its wait start, which +/// the chart draws exactly. A head that changed means a moving queue, whose delay hovers and is +/// held; so is a first sample, which cannot tell yet and must not draw a climb that never was. +fn delay_sample( + last_head_since: Option, + stat: &windmill_common::queue::QueueStat, +) -> QueueSample { + match last_head_since { + Some(since) if (since - stat.head_since).abs() < QUEUE_DELAY_SAME_HEAD_SECS => { + QueueSample::Climbing { since: stat.head_since } + } + _ => QueueSample::Held(stat.delay), + } +} + +/// Whether the next delay sample is drawn differently from the last one even at the same value: +/// a climb whose head left would otherwise go on climbing from the old head, and a held delay +/// whose head stayed would stay flat while the wait grows. +fn redraws(last: QueueSample, next: QueueSample) -> bool { + matches!(last, QueueSample::Climbing { .. }) != matches!(next, QueueSample::Climbing { .. }) +} + +/// Whether a reading deserves a row of its own, given the last one stored for that metric: +/// the value it draws now and how many seconds ago it was written. `current` is `None` once +/// the tag has no backlog left; `redraws` is set when the reading must be drawn differently. +fn should_store( + prefix: &str, + last: Option<(f64, f64)>, + current: Option, + redraws: bool, +) -> bool { + let Some((last_value, age)) = last else { + // Nothing comparable within the lookback window: a tag that just backed up needs a + // first sample, one that was already gone needs nothing. + return current.is_some(); + }; + let Some(current) = current else { + // The tag drained. One zero pins where the line drops; after that the metric matches + // and goes quiet, then falls out of the lookback window entirely. + return last_value != 0.0; + }; + if age >= QUEUE_METRIC_HEARTBEAT_SECS { + return true; + } + age >= QUEUE_METRIC_MIN_INTERVAL_SECS + && (redraws + || if prefix == QUEUE_COUNT_PREFIX { + last_value != current + } else { + (current - last_value).abs() > last_value.abs() * QUEUE_DELAY_TOLERANCE + }) +} + +#[cfg(test)] +mod queue_metric_sampling { + use super::*; + + const RECENT: f64 = QUEUE_METRIC_MIN_INTERVAL_SECS + 1.0; + + #[test] + fn a_holding_backlog_writes_only_on_the_heartbeat() { + let held = Some((3.0, RECENT)); + assert!(!should_store(QUEUE_COUNT_PREFIX, held, Some(3.0), false)); + let due = Some((3.0, QUEUE_METRIC_HEARTBEAT_SECS)); + assert!(should_store(QUEUE_COUNT_PREFIX, due, Some(3.0), false)); + // A held delay hovers, so only a move past the tolerance counts as a change. + let delay = Some((100.0, RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, delay, Some(105.0), false)); + assert!(should_store(QUEUE_DELAY_PREFIX, delay, Some(120.0), false)); + } + + #[test] + fn a_drained_tag_writes_one_zero_then_stops() { + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + None, + false + )); + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((0.0, RECENT)), + None, + false + )); + // Including once the heartbeat is due: a tag that is gone stays silent. + let gone = Some((0.0, QUEUE_METRIC_STALE_SECS)); + assert!(!should_store(QUEUE_COUNT_PREFIX, gone, None, false)); + assert!(!should_store(QUEUE_COUNT_PREFIX, None, None, false)); + } + + #[test] + fn a_change_waits_for_the_minimum_interval() { + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, 1.0)), + Some(9.0), + false + )); + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + Some(9.0), + false + )); + // A tag that has just backed up is recorded at once. + assert!(should_store(QUEUE_COUNT_PREFIX, None, Some(9.0), false)); + } + + #[test] + fn a_delay_climbs_while_the_same_job_stays_at_the_head() { + let stat = windmill_common::queue::QueueStat { count: 3, delay: 330.0, head_since: 1000.0 }; + // A held sample written at 1320 saw the same head: it switches to climbing at once, + // although the delay has not moved past the tolerance yet. + let first = QueueSample::Held(320.0); + let climbing = delay_sample(Some(first.head_since(1320.0)), &stat); + assert_eq!(climbing, QueueSample::Climbing { since: 1000.0 }); + assert!(redraws(first, climbing)); + let drawn = Some((first.value_at(1330.0), RECENT)); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(stat.delay), + true + )); + // Stored climbing, it draws the delay exactly: nothing more until the heartbeat. + let drawn = Some((climbing.value_at(1600.0), RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, drawn, Some(600.0), false)); + assert_eq!(delay_sample(None, &stat), QueueSample::Held(330.0)); + } + + #[test] + fn a_climb_whose_head_left_is_held_even_within_the_tolerance() { + // The head waiting since 0 left at 3600 for one queued at 100: 3500s is within 10% of + // the 3600s the climb draws, but kept, the climb would go on from the old head. + let moved = + windmill_common::queue::QueueStat { count: 2, delay: 3500.0, head_since: 100.0 }; + let climbing = QueueSample::Climbing { since: 0.0 }; + let next = delay_sample(Some(climbing.head_since(3000.0)), &moved); + assert_eq!(next, QueueSample::Held(3500.0)); + assert!(redraws(climbing, next)); + let drawn = Some((climbing.value_at(3600.0), RECENT)); + assert!(!should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + false + )); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + true + )); + } +} + +/// When this server last sampled the queue into `metrics`, in Unix milliseconds. It only paces +/// how often the queue is scanned for that; whether a sample earns a row is decided from what +/// is already stored. Servers sampling in the same instant can each write it, and the duplicate +/// draws the same. +static LAST_QUEUE_SAMPLE_MS: std::sync::atomic::AtomicI64 = std::sync::atomic::AtomicI64::new(0); + +pub async fn expose_queue_metrics(db: &Pool) { // clean queue metrics older than 14 days sqlx::query!( "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'" @@ -5083,6 +5435,131 @@ pub async fn expose_queue_metrics(db: &Pool) { .execute(db) .await .ok(); + + let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); + let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let now_ms = chrono::Utc::now().timestamp_millis(); + let save_metrics = now_ms - LAST_QUEUE_SAMPLE_MS.load(Ordering::Relaxed) + >= (QUEUE_METRIC_MIN_INTERVAL_SECS * 1000.0) as i64; + if !(metrics_enabled || otel_enabled || save_metrics) { + return; + } + + // Single DB query for running counts, shared by Prometheus and OTel. It runs ahead of the + // backlog read below, which gives up on the rest of the round when it fails. + let otel_running = otel_enabled; + #[cfg(feature = "prometheus")] + let need_running_counts = metrics_enabled || otel_running; + #[cfg(not(feature = "prometheus"))] + let need_running_counts = otel_running; + + if need_running_counts { + let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_running { + for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + otel_set_queue_running_count(q, 0); + } + } + } + + #[allow(unused_mut, unused_variables)] + let mut running_tags_to_watch: Vec = vec![]; + #[allow(unused_mut, unused_variables)] + let mut otel_running_tags_to_watch: Vec = vec![]; + for (tag, count) in &queue_running_counts { + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); + metric.set(*count as i64); + running_tags_to_watch.push(tag.to_string()); + } + + if otel_running { + otel_set_queue_running_count(tag, *count as i64); + otel_running_tags_to_watch.push(tag.to_string()); + } + } + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = running_tags_to_watch; + } + if otel_running { + let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = otel_running_tags_to_watch; + } + } + + let queue_stats = match windmill_common::queue::get_queue_stats(db).await { + Ok(queue_stats) => queue_stats, + Err(e) => { + tracing::error!("Failed to read queue stats: {e:#}"); + return; + } + }; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + (*QUEUE_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_enabled { + for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + otel_set_queue_count(q, 0); + } + } + } + + #[allow(unused_mut)] + let mut tags_to_watch = vec![]; + #[allow(unused_mut)] + let mut otel_tags_to_watch = vec![]; + for (tag, stat) in queue_stats.iter() { + let count = stat.count; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_COUNT).with_label_values(&[tag]); + metric.set(count as i64); + tags_to_watch.push(tag.to_string()); + } + + if otel_enabled { + otel_tags_to_watch.push(tag.to_string()); + } + otel_set_queue_count(tag, count as i64); + } + + if save_metrics { + LAST_QUEUE_SAMPLE_MS.store(now_ms, Ordering::Relaxed); + save_queue_metrics(db, &queue_stats).await; + } + + if metrics_enabled { + let mut w = QUEUE_COUNT_TAGS.write().await; + *w = tags_to_watch; + } + if otel_enabled { + let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; + *w = otel_tags_to_watch; + } } pub async fn reload_smtp_config(db: &Pool) { @@ -5117,6 +5594,7 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b .dedicated_workers .as_ref() .is_some_and(|dws| !dws.is_empty()); + if **wc != config || has_dedicated { if kill_if_change { if has_dedicated @@ -5164,6 +5642,37 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b store_pull_query(&config).await; WORKER_CONFIG.store(std::sync::Arc::new(config)); } + + // After the store, so a retry that wakes mid-build reads the config being applied + // rather than the one it replaced. Unconditional rather than gated on the value + // changing, so that a pass triggered by anything else — a license-plan change, most + // of all — still re-evaluates the entitlement. + #[cfg(feature = "parquet")] + reload_cache_object_store_override_with_retry(db).await; + } +} + +/// Apply this worker group's dependency-cache object store, retrying once shortly after a build +/// that failed for a reason that may pass — the periodic settings reload behind it is 12h apart, +/// which is a long time for a whole group to cache nothing but locally. +#[cfg(feature = "parquet")] +pub async fn reload_cache_object_store_override_with_retry(db: &DB) { + let settings = WORKER_CONFIG.load().object_store_cache_config.clone(); + if matches!( + windmill_object_store::reload_cache_object_store_override(db, settings).await, + ObjectStoreReload::Later + ) { + let db = db.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(10)).await; + if windmill_object_store::cache_object_store_override_failed().await { + // Re-read rather than reuse: the group config may have changed while we slept, + // and installing the settings this retry was born with would pin the worker to a + // store the group no longer asks for. + let settings = WORKER_CONFIG.load().object_store_cache_config.clone(); + windmill_object_store::reload_cache_object_store_override(&db, settings).await; + } + }); } } @@ -6514,6 +7023,34 @@ pub fn apply_app_workspaced_route_setting(app_workspaced_route: Option error::Result<()> { + let v = + load_value_from_global_settings(conn, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING).await?; + apply_http_route_default_allowed_origins_setting(v) +} + +pub fn apply_http_route_default_allowed_origins_setting( + value: Option, +) -> error::Result<()> { + // A bad value leaves whatever is already loaded in place rather than + // reverting to no restriction. On the boot path that is still the empty + // default, so what keeps a stored typo from widening CORS instance-wide is + // write-time validation, not this. + let origins = match parse_allowed_origins_setting(value.as_ref()) { + Ok(origins) => origins, + Err(err) => { + tracing::error!( + "Invalid {} setting, keeping the previous value: {err:#}", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ); + return Ok(()); + } + }; + + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.store(std::sync::Arc::new(origins)); + Ok(()) +} + pub async fn reload_http_route_workspaced_route_setting(conn: &DB) -> error::Result<()> { let v = load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?; apply_http_route_workspaced_route_setting(conn, v).await @@ -6579,7 +7116,6 @@ pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<( Ok(()) } - pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { let v = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; apply_jwt_secret_setting(db, v).await diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index dfd0305e57..41dd70ca93 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -70,11 +70,15 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) -dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) +dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) +dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) +dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char), id(bigint) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) + FK: (workspace_id) -> workspace(id) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) FK: (workspace_id) -> workspace(id) @@ -84,7 +88,7 @@ debounce_key: key(char), job_id(uuid), previous_job_id(uuid), first_started_at(t debounce_stale_data: job_id(uuid), to_relock(text[]) debouncing_settings: hash(bigint), debounce_key(char), debounce_delay_s(int), max_total_debouncing_time(int), max_total_debounces_amount(int), debounce_args_to_accumulate(text[]) dependency_map: workspace_id(char), importer_path(char), importer_kind(importer_kind), imported_path(char), importer_node_id(char) -deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid) +deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid), id(bigint) FK: (workspace_id) -> workspace(id) draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at(ts) FK: (workspace_id) -> workspace(id) @@ -109,13 +113,19 @@ folder: name(char), workspace_id(char), display_name(char), owners(char), extra_ folder_permission_history: id(bigint), workspace_id(char), folder_name(char), changed_by(char), changed_at(ts), change_type(char), affected(char) FK: (workspace_id, folder_name) -> folder(workspace_id, name) gcp_trigger: gcp_resource_path(char), topic_id(char), subscription_id(char), delivery_type(delivery_mode), delivery_config(jsonb), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), subscription_mode(gcp_subscription_mode), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), auto_acknowledge_msg(bool), ack_deadline(int), mode(trigger_mode), labels(text[]) +git_sync_ci_test_check: workspace_id(char), head_sha(char), head_ref(char), poster_workspace_id(char), repo_url(text), repo_resource_path(char), check_run_id(bigint), created_at(timestamptz), concluded(bool), conclusion(text), concluded_at(timestamptz), github_posted(bool) + FK: (workspace_id) -> workspace(id) + FK: (poster_workspace_id) -> workspace(id) +git_sync_synced_head: workspace_id(char), repo_resource_path(char), branch(char), sha(char), source(char), job_id(uuid), synced_at(timestamptz), tests_dispatched_at(timestamptz), ci_test_job_ids(uuid[]) + FK: (workspace_id) -> workspace(id) global_settings: name(char), value(jsonb), updated_at(ts) +guest_activity: email(char), workspace_id(char), day(date), last_seen_at(timestamptz), jwt_entry(bool) group_: workspace_id(char), name(char), summary(text), extra_perms(jsonb) FK: (workspace_id) -> workspace(id) group_permission_history: id(bigint), workspace_id(char), group_name(char), changed_by(char), changed_at(ts), change_type(char), member_affected(char) FK: (workspace_id, group_name) -> group_(workspace_id, name) healthchecks: id(bigint), check_type(text), healthy(bool), created_at(ts) -http_trigger: path(char), route_path(char), route_path_key(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), authentication_method(authentication_method), http_method(http_method), static_asset_config(jsonb), is_static_website(bool), workspaced_route(bool), wrap_body(bool), raw_string(bool), authentication_resource_path(char), summary(char), description(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), request_type(request_type), mode(trigger_mode), labels(text[]) +http_trigger: path(char), route_path(char), route_path_key(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), authentication_method(authentication_method), http_method(http_method), static_asset_config(jsonb), is_static_website(bool), workspaced_route(bool), wrap_body(bool), raw_string(bool), allowed_origins(text[]), authentication_resource_path(char), summary(char), description(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), request_type(request_type), mode(trigger_mode), labels(text[]) input: id(uuid), workspace_id(char), runnable_id(char), runnable_type(runnable_type), name(text), args(jsonb), created_at(ts), created_by(char), is_public(bool) FK: (workspace_id) -> workspace(id) instance_group: name(char), summary(char), id(char), scim_display_name(char), external_id(char) @@ -142,9 +152,9 @@ mcp_oauth_refresh_token: id(bigint), refresh_token(char), access_token_hash(char mcp_oauth_server_client: client_id(char), client_name(char), redirect_uris(text[]), created_at(ts) mcp_oauth_server_code: code(char), client_id(char), user_email(char), workspace_id(char), scopes(text[]), redirect_uri(text), code_challenge(char), code_challenge_method(char), created_at(ts), expires_at(ts) FK: (client_id) -> mcp_oauth_server_client(client_id) -metrics: id(char), value(jsonb), created_at(ts) +metrics: id(char), value(jsonb), created_at(ts), row_id(bigint) mqtt_trigger: mqtt_resource_path(char), subscribe_topics(jsonb[]), client_version(mqtt_client_version), v5_config(jsonb), v3_config(jsonb), client_id(char), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) -native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts) +native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts), enabled(bool) FK: (workspace_id) -> workspace(id) nats_trigger: path(char), nats_resource_path(char), subjects(char), stream_name(char), consumer_name(char), use_jetstream(bool), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) FK: (workspace_id) -> workspace(id) @@ -220,9 +230,9 @@ workspace_key: workspace_id(char), kind(workspace_key_kind), key(char) FK: (workspace_id) -> workspace(id) workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_groups(text[]), bypass_users(text[]), created_at(ts) FK: (workspace_id) -> workspace(id) -workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char) +workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index 791556b035..4d6fffa346 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -293,6 +293,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> let lfs_config = json!({ "type": "FilesystemStorage", "root_path": storage_root, + "volume_storage": "primary", "public_resource": null, "advanced_permissions": null }); @@ -306,7 +307,11 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> .await?; // 2. Pre-populate the volume with a file - let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); std::fs::create_dir_all(&vol_dir)?; std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; @@ -343,6 +348,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> // 4. GET /file/* — download the existing file let resp = http .get(format!("{vol_base}/file/hello.txt")) + .query(&[("worker_name", "test-worker-1")]) .send() .await?; assert!( @@ -360,6 +366,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> // 5. PUT /file/* — upload a new file let resp = http .put(format!("{vol_base}/file/output.txt")) + .query(&[("worker_name", "test-worker-1")]) .body(b"written by agent worker".to_vec()) .send() .await?; @@ -432,7 +439,8 @@ async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow: "type": "FilesystemStorage", "root_path": storage_root, "public_resource": null, - "advanced_permissions": null + "advanced_permissions": null, + "volume_storage": "primary" }); sqlx::query!( @@ -444,20 +452,24 @@ async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow: .await?; // 2. Pre-populate the volume with a file - let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); std::fs::create_dir_all(&vol_dir)?; std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; // 3. Push the job, then run worker with HTTP connection (bun tag) - let code = r#"// volume: test-vol /tmp/data + let code = r#"// volume: test-vol data import { readFileSync, writeFileSync, existsSync } from "fs"; export function main() { - const content = readFileSync("/tmp/data/hello.txt", "utf-8"); - writeFileSync("/tmp/data/output.txt", "written by agent worker"); + const content = readFileSync("data/hello.txt", "utf-8"); + writeFileSync("data/output.txt", "written by agent worker"); return { read_content: content, - output_exists: existsSync("/tmp/data/output.txt"), + output_exists: existsSync("data/output.txt"), }; }"#; @@ -528,6 +540,7 @@ async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result< let lfs_config = json!({ "type": "FilesystemStorage", "root_path": storage_root, + "volume_storage": "primary", "public_resource": null, "advanced_permissions": null }); diff --git a/backend/tests/app_guest_allowance.rs b/backend/tests/app_guest_allowance.rs new file mode 100644 index 0000000000..88626b1c81 --- /dev/null +++ b/backend/tests/app_guest_allowance.rs @@ -0,0 +1,159 @@ +//! The guest allowance: free up to `FREE_GUESTS_PER_WINDOW` distinct emails over the +//! trailing window. Past it, a hard-capped instance (Community, Pro) refuses a stranger +//! and lets a returning guest back in; a metered one (Enterprise) admits everyone and +//! counts seats. Its own binary: the plan is read from a process-wide key id that this +//! test flips, which no test sharing the process could tolerate. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::{FREE_GUESTS_PER_WINDOW, GUEST_WINDOW_DAYS}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +/// Community and Pro are capped, Enterprise is metered. Only a build with both +/// `private` (the key id) and `enterprise` (the plan read) can meter; every other build +/// is capped whatever this says. +fn set_plan(pro: bool) { + #[cfg(feature = "private")] + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new( + if pro { "test_pro" } else { "" }.to_string(), + )); + let _ = pro; +} + +async fn mint(db: &Pool, email: &str) -> windmill_common::error::Result { + let mut tx = db.begin().await.unwrap(); + let minted = windmill_api_users::users::create_guest_session_token( + email, + "test-workspace", + APP_PATH, + &mut tx, + tower_cookies::Cookies::default(), + ) + .await; + tx.commit().await.unwrap(); + minted +} + +#[sqlx::test(fixtures("base"))] +async fn the_allowance_caps_strangers_and_meters_an_enterprise_plan( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables_v2": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + // The whole allowance, used yesterday: still in the window, and a day the mint + // does not write, so a row dated today can only be the mint's own. + sqlx::query( + "INSERT INTO guest_activity (email, workspace_id, day) + SELECT 'g' || i || '@example.com', 'test-workspace', CURRENT_DATE - 1 + FROM generate_series(1, $1) AS i", + ) + .bind(FREE_GUESTS_PER_WINDOW) + .execute(&db) + .await?; + + set_plan(true); + let refused = mint(&db, "stranger@example.com").await.unwrap_err(); + assert!( + matches!(&refused, windmill_common::error::Error::PermissionDenied(m) + if m.contains(&format!("limit of {FREE_GUESTS_PER_WINDOW} guests over {GUEST_WINDOW_DAYS} days"))), + "a stranger past the allowance is refused with the message the visitor reads: {refused:?}" + ); + mint(&db, "g1@example.com") + .await + .expect("a guest already in the window is let back in"); + let recorded: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM guest_activity + WHERE email = 'g1@example.com' AND workspace_id = 'test-workspace' + AND day = CURRENT_DATE)", + ) + .fetch_one(&db) + .await?; + assert!( + recorded, + "the mint writes today's guest_activity row, the allowance's unit" + ); + + let list: serde_json::Value = authed( + client().get(format!( + "http://localhost:{port}/api/users/guests?per_page=5" + )), + ADMIN_TOKEN, + ) + .send() + .await? + .json() + .await?; + assert_eq!(list["usage"]["guest_count"], FREE_GUESTS_PER_WINDOW); + assert_eq!(list["usage"]["metered"], false); + assert_eq!(list["usage"]["guest_seats"], 0); + assert_eq!(list["guests"].as_array().map(Vec::len), Some(5)); + assert_eq!(list["guests"][0]["workspaces"], json!(["test-workspace"])); + let usage: serde_json::Value = authed( + client().get(format!("{ws}/workspaces/guest_usage")), + ADMIN_TOKEN, + ) + .send() + .await? + .json() + .await?; + assert_eq!(usage["guest_count"], FREE_GUESTS_PER_WINDOW); + + #[cfg(all(feature = "private", feature = "enterprise"))] + { + set_plan(false); + mint(&db, "stranger@example.com") + .await + .expect("a metered plan admits past the allowance"); + let usage: serde_json::Value = authed( + client().get(format!("{ws}/workspaces/guest_usage")), + ADMIN_TOKEN, + ) + .send() + .await? + .json() + .await?; + assert_eq!(usage["guest_count"], FREE_GUESTS_PER_WINDOW + 1); + assert_eq!(usage["metered"], true); + assert_eq!(usage["billable_guests"], 1); + assert_eq!( + usage["guest_seats"], 1, + "one guest past the allowance is a whole seat" + ); + } + + Ok(()) +} diff --git a/backend/tests/app_guest_cloud_hosted.rs b/backend/tests/app_guest_cloud_hosted.rs new file mode 100644 index 0000000000..7ab4145a65 --- /dev/null +++ b/backend/tests/app_guest_cloud_hosted.rs @@ -0,0 +1,168 @@ +//! Guests are unavailable on the shared cloud (`CLOUD_HOSTED`). +//! +//! One test in its own binary on purpose: `CLOUD_HOSTED` is read once into a +//! `lazy_static`, so it must be set before anything reads it and cannot be unset for a +//! sibling test in the same process. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const GUEST_TOKEN: &str = "GUEST_SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn the_cloud_admits_no_guest(db: Pool) -> anyhow::Result<()> { + // Before the server starts, so the flag is what the whole process sees. + unsafe { std::env::set_var("CLOUD_HOSTED", "true") }; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // The workspace switch cannot be turned on, so no policy can lean on it. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("self-hosted"), + "the refusal must name what guests need" + ); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "an app cannot be deployed to guests"); + + // Nor can a key be configured for the JWT way in — the refusal lands before the + // outbound JWKS fetch it would otherwise make. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({ "jwks_url": "https://issuer.example.com/.well-known/jwks.json" })) + .send() + .await?; + assert_eq!(resp.status(), 400, "a guest JWT key cannot be configured"); + // Clearing one stays allowed: a key nobody can use is still worth removing. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // An app already stored in guest mode — pushed by git-sync, or deployed before the + // instance became a cloud one — advertises no entry either. + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + sqlx::query( + "UPDATE app SET policy = jsonb_set(policy, '{execution_mode}', '\"guest\"') + WHERE path = $1 AND workspace_id = 'test-workspace'", + ) + .bind(APP_PATH) + .execute(&db) + .await?; + sqlx::query("UPDATE workspace_settings SET guest_access_enabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + + // Deploying it again is not refused: only widening an app into guests is, so a + // git-sync push of one already stored that way keeps working (and keeps being inert). + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "an app already stored in guest mode must stay deployable: {}", + resp.text().await? + ); + + let resp = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "reading the share secret must succeed"); + let secret: String = resp.text().await?; + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a guest app must not advertise entry where guests are unavailable" + ); + + // And a session issued before the instance became a cloud one stops on its next + // request: the door re-reads the switch, so the credential itself is not enough. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'GUEST_SECR', $2, 'guest@example.com', + 'guest_session', $3, 'test-workspace', now() + interval '8 hours')", + ) + .bind(GUEST_TOKEN.as_bytes()) + .bind(GUEST_TOKEN) + .bind(vec![ + "guest".to_string(), + "users:read".to_string(), + format!("apps:read:{APP_PATH}"), + format!("apps:run:{APP_PATH}"), + ]) + .execute(&db) + .await?; + // `whoami` is where an admitted guest resolves as `role: guest`, so a 401 here is + // the door refusing the credential rather than a route saying no. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest session must not authenticate where guests are unavailable" + ); + + Ok(()) +} diff --git a/backend/tests/app_guest_execution_mode.rs b/backend/tests/app_guest_execution_mode.rs new file mode 100644 index 0000000000..7af2e0794e --- /dev/null +++ b/backend/tests/app_guest_execution_mode.rs @@ -0,0 +1,1092 @@ +//! Tests for the `guest` app execution mode. +//! +//! A guest (`ExecutionMode::Guest`) has no account and so no ACL of its own: its +//! token's scopes are its entire grant. These tests pin the three things that would +//! silently undo it: +//! +//! * what makes a token a guest — the server-minted label, never a scope anyone +//! could type into `users/tokens/create`; +//! * the confinement — a guest reaches the one app it was let in for and nothing +//! else; +//! * the switches — an app's own `execution_mode: guest` is inert unless the +//! workspace and the instance allow guests, checked at the door rather than only +//! where a policy is written (git-sync and the CLI push policies past every UI); +//! the allowance on top of them has a binary of its own. +//! +//! The token is inserted directly: how a guest session is minted is the identity +//! provider's business (EE), what one can do is this file's. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const GUEST_TOKEN: &str = "GUEST_SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +async fn enable_guests(port: u16, ws: &str) -> anyhow::Result<()> { + authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/workspaces/edit_guest_access" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + Ok(()) +} + +fn guest_scopes() -> Vec { + vec![ + "guest".to_string(), + "jobs:read".to_string(), + "resources:run".to_string(), + "users:read".to_string(), + "folders:read".to_string(), + format!("apps:read:{APP_PATH}"), + format!("apps:run:{APP_PATH}"), + ] +} + +/// Insert a guest session for `test-workspace`, scoped to `APP_PATH`. Mirrors +/// `create_guest_session_token`: the server-minted label, the narrow reads, the two +/// path-scoped app grants, the workspace pin, and an expiry — a derived token's +/// lifetime is capped at it, so a guest session without one cannot mint. +async fn insert_guest_token(db: &Pool, workspace: &str) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'GUEST_SECR', $2, 'guest@example.com', + 'guest_session', $3, $4, now() + interval '8 hours')", + ) + .bind(GUEST_TOKEN.as_bytes()) + .bind(GUEST_TOKEN) + .bind(guest_scopes()) + .bind(workspace) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn guest_session_is_confined_to_its_app(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + insert_guest_token(&db, "test-workspace").await?; + + // Its own identity resolves, and reports the role rather than falling through to + // the non-member branch that hands out a `superadmin` shape. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "guest whoami must resolve"); + let me: serde_json::Value = resp.json().await?; + assert_eq!( + me["role"], + json!("guest"), + "guest must not read as superadmin" + ); + assert_eq!(me["operator"], json!(true)); + assert_eq!(me["is_admin"], json!(false)); + + // `resources/list_names` and the type schemas stay open — a guest drives an app, + // and app pickers need them — so the line to pin is the value-returning route. + for route in [ + "jobs/list", + "scripts/list", + "flows/list", + "variables/list", + "resources/get_value/u/test-user/secret", + "apps/list", + ] { + let resp = authed(client().get(format!("{ws}/{route}")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "guest must be denied {route}, got {}", + resp.status() + ); + } + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn guest_token_does_not_cross_workspaces(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // A second workspace with an app at the SAME path: without the token's workspace + // pin, `apps:run:` would unlock it too, since a path is not unique across + // workspaces. + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ('other-ws', 'other-ws', 'test-user')", + ) + .execute(&db) + .await?; + sqlx::query("INSERT INTO workspace_settings (workspace_id) VALUES ('other-ws')") + .execute(&db) + .await?; + + insert_guest_token(&db, "test-workspace").await?; + + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/other-ws/apps/get/p/{APP_PATH}" + )), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest token pinned to one workspace must not authenticate against another" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn guest_entry_needs_both_the_app_mode_and_the_workspace_switch( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + + let secret: String = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + + // The app says guest, the workspace has not opted in: inert. + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a guest app in a workspace that has not enabled guests must not advertise entry" + ); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + + // Unauthenticated on purpose: this is what a signed-out visitor reads. + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let entry: serde_json::Value = resp.json().await?; + assert_eq!(entry["app_path"], json!(APP_PATH)); + + // Turning the switch back off closes the door again even though the app's own + // policy is unchanged. + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": false })) + .send() + .await?; + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "turning guests off must stop advertising entry for an app already set to guest" + ); + + Ok(()) +} + +/// The guest grant is the server-minted label, never the `guest` scope. Scopes on a +/// user-created token are whatever the caller typed, so if the scope granted anything +/// then any member of any workspace could mint themselves non-member access to every +/// guest-mode app on the instance. +#[sqlx::test(fixtures("base"))] +async fn a_self_declared_guest_scope_grants_nothing(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // `users/tokens/create` must refuse the label outright... + let resp = authed( + client().post(format!("http://localhost:{port}/api/users/tokens/create")), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "guest_session", "scopes": guest_scopes() })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "the guest session label must be server-minted only" + ); + + // ...and so must relabelling an ordinary token into it, or the pin-less user + // token would become a guest session that authenticates in every workspace. + let resp = authed( + client().post(format!("http://localhost:{port}/api/users/tokens/create")), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "mine", "scopes": guest_scopes() })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let prefix: String = sqlx::query_scalar( + "SELECT token_prefix FROM token WHERE email = 'test@windmill.dev' AND label = 'mine'", + ) + .fetch_one(&db) + .await?; + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/users/tokens/update_label/{prefix}" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "guest_session" })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "relabelling into the guest namespace must be refused: {}", + resp.text().await? + ); + + // ...and a token that carries the scopes under any other label authenticates as + // nothing in a workspace its owner is not a member of. + // An email with no `usr` row anywhere: exactly the identity the guest arm exists + // to admit, and the one a forged scope must not admit. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes) + VALUES (encode(sha256($1::bytea), 'hex'), 'FORGED_SCO', $2, 'outsider@example.com', + 'forged', $3)", + ) + .bind(b"FORGED_SCOPES".as_slice()) + .bind("FORGED_SCOPES") + .bind(guest_scopes()) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{ws}/users/whoami")), "FORGED_SCOPES") + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "declaring the guest scope must not turn a non-member into an identity" + ); + + Ok(()) +} + +/// A guest-mode policy that names one runnable, so an `execute_component` request +/// gets past the triggerables lookup and reaches the guest gate. `sandbox` is what +/// makes the embed-token endpoint actually mint a token. +fn guest_app_with_runnable(path: &str, sandbox: bool) -> serde_json::Value { + app_with_runnable(path, "guest", sandbox) +} + +fn app_with_runnable(path: &str, execution_mode: &str, sandbox: bool) -> serde_json::Value { + json!({ + "path": path, + "summary": "App", + "value": {}, + "policy": { + "execution_mode": execution_mode, + "sandbox": sandbox, + "triggerables_v2": { + "script/u/test-user/noop": { "static_inputs": {}, "one_of_inputs": {} } + } + } + }) +} + +fn execute(port: u16, ws: &str, app: &str, token: &str) -> reqwest::RequestBuilder { + authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/apps_u/execute_component/{app}" + )), + token, + ) + .json(&json!({ + "component": "a", + "path": "script/u/test-user/noop", + "args": {} + })) +} + +/// The workspace switch is enforced at the auth door for every guest request, not +/// remembered per handler. This is what stands between a `guest` policy pushed by +/// git-sync and execution once an admin has turned guests off — and it closes the +/// app to sessions already issued. +#[sqlx::test(fixtures("base"))] +async fn the_door_re_checks_the_workspace_switch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; + + // Switch off: the session does not authenticate at all, even though the app's + // policy says guest and the session was (in this fixture) issued regardless. On + // the authed route that is a 401; on the optional-auth run route the rejected + // token reads as no token, and a guest-mode app then refuses the anonymous + // caller — a denial either way. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest must not authenticate while guests are off" + ); + let resp = execute(port, "test-workspace", APP_PATH, GUEST_TOKEN) + .send() + .await?; + assert!( + resp.status().is_client_error() && resp.status() != 404, + "a guest must not run while guests are off, got {}", + resp.status() + ); + + // Switch on: through the door. What follows the run is the runnable lookup, + // which fails on the nonexistent script — the point is that it is no longer a + // denial. + enable_guests(port, "test-workspace").await?; + let resp = execute(port, "test-workspace", APP_PATH, GUEST_TOKEN) + .send() + .await?; + assert!( + resp.status() != 401 && resp.status() != 403, + "with guests on, the door must let the run through: {}", + resp.status() + ); + + Ok(()) +} + +/// The path scope is what keeps a guest to the one app it was let in for: the route +/// layer is resource-blind for `apps:run`, so this line is drawn in the handler. +#[sqlx::test(fixtures("base"))] +async fn guest_cannot_run_another_guest_app(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let other = "u/test-user/other_guest_app"; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(other, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; // scoped to APP_PATH, not `other` + + let resp = execute(port, "test-workspace", other, GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "a guest session scoped to one app must not run another, even one open to guests" + ); + + Ok(()) +} + +/// The app path is spliced into the session's scopes, whose grammar reserves `:`, `,` +/// and `*`: a path carrying one would scope the guest to more than the one app it was +/// let in for, so the mint refuses it before anything else. Anything else in a path +/// (spaces, `@`) is literal to that grammar and stays admissible. +#[sqlx::test(fixtures("base"))] +async fn a_scope_metacharacter_in_the_app_path_is_refused( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let mint = |path: &'static str| { + let db = db.clone(); + async move { + let mut tx = db.begin().await?; + let minted = windmill_api_users::users::create_guest_session_token( + "guest@example.com", + "test-workspace", + path, + &mut tx, + tower_cookies::Cookies::default(), + ) + .await; + anyhow::Ok(minted) + } + }; + for path in [ + "u/test-user/entry,u/test-user/hidden", + "u/test-user/*", + "u/test-user/entry:run", + ] { + let minted = mint(path).await?; + assert!( + matches!(minted, Err(windmill_common::error::Error::BadRequest(ref m)) if m.contains("cannot be scoped")), + "{path}: {minted:?}" + ); + } + for path in ["u/test-user/My App", "u/admin@windmill.dev/x"] { + let minted = mint(path).await?; + assert!( + !matches!(minted, Err(windmill_common::error::Error::BadRequest(ref m)) if m.contains("cannot be scoped")), + "{path} is literal to the scope grammar and must get past the guard: {minted:?}" + ); + } + Ok(()) +} + +/// A guest reads the jobs it launched and nothing else: with no membership behind it, +/// it must stop where an app embed token stops, before the share-token and ACL grants +/// a member would get, and with the same "not found" so it cannot probe for jobs. +#[sqlx::test(fixtures("base"))] +async fn a_guest_cannot_read_a_job_it_did_not_launch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + insert_guest_token(&db, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/scripts/create")), ADMIN_TOKEN) + .json(&json!({ + "path": "u/test-user/noop", + "summary": "", + "description": "", + "content": "echo 42", + "language": "bash", + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let resp = authed( + client().post(format!("{ws}/jobs/run/p/u/test-user/noop")), + ADMIN_TOKEN, + ) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let job_id = resp.text().await?; + + let resp = authed( + client().get(format!("{ws}/jobs_u/getupdate/{job_id}")), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "another caller's job is not found for a guest: {}", + resp.text().await? + ); + + Ok(()) +} + +/// Guests mode cannot land on a path the scope grammar cannot hold, however it gets +/// there: set at creation, set on update, or a rename of an app already in that mode. +#[sqlx::test(fixtures("base"))] +async fn guests_mode_needs_a_scopable_path(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable("u/test-user/a:b", false)) + .send() + .await?; + assert_eq!(resp.status(), 400, "created into Guests on a `:` path"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ "path": "u/test-user/a,b" })) + .send() + .await?; + assert_eq!(resp.status(), 400, "renamed to a `,` path while in Guests"); + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ "path": "u/test-user/My App" })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a space is literal: {}", + resp.text().await? + ); + + // Set on update: an app that already sits on such a path cannot be switched. + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": "u/test-user/x:y", + "summary": "App", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables_v2": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let resp = authed( + client().post(format!("{ws}/apps/update/u/test-user/x:y")), + ADMIN_TOKEN, + ) + .json(&json!({ "policy": { "execution_mode": "guest", "triggerables_v2": {} } })) + .send() + .await?; + assert_eq!(resp.status(), 400, "switched to Guests on a `:` path"); + + Ok(()) +} + +/// Renaming a workspace copies its settings; the guest switch and the guest JWT key must +/// travel with them, or the rename silently shuts every guest app or drops the key. +#[sqlx::test(fixtures("base"))] +async fn a_workspace_rename_keeps_the_guest_switch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + enable_guests(port, "test-workspace").await?; + sqlx::query( + "INSERT INTO guest_activity (email, workspace_id, day) + VALUES ('guest@example.com', 'test-workspace', CURRENT_DATE)", + ) + .execute(&db) + .await?; + sqlx::query( + "UPDATE workspace_settings SET guest_jwt_public_key = 'test-pem-key' WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/change_workspace_id" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "new_id": "test-workspace-2", "new_name": "Test workspace 2" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let enabled: bool = sqlx::query_scalar( + "SELECT guest_access_enabled FROM workspace_settings WHERE workspace_id = 'test-workspace-2'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "the guest switch travels with the workspace"); + let jwt_key: Option = sqlx::query_scalar( + "SELECT guest_jwt_public_key FROM workspace_settings WHERE workspace_id = 'test-workspace-2'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + jwt_key.as_deref(), + Some("test-pem-key"), + "the guest JWT key travels with the workspace" + ); + let moved: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM guest_activity WHERE workspace_id = 'test-workspace-2') + AND NOT EXISTS(SELECT 1 FROM guest_activity WHERE workspace_id = 'test-workspace')", + ) + .fetch_one(&db) + .await?; + assert!(moved, "the guests seen in the workspace follow its new id"); + + Ok(()) +} + +/// The superadmin switch sits above every workspace's: off, no guest session stands and +/// no app discovers as open, whatever the workspace and the app say. +#[sqlx::test(fixtures("base"))] +async fn the_instance_switch_closes_every_workspace(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; + let set_instance_switch = |disabled: bool| { + authed( + client().post(format!( + "http://localhost:{port}/api/settings/global/guest_access_disabled" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "value": disabled })) + .send() + }; + + let secret: String = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + + set_instance_switch(true).await?.error_for_status()?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "the instance switch closes an issued session" + ); + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "and nothing discovers as open to guests" + ); + + set_instance_switch(false).await?.error_for_status()?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "back on, the session stands again"); + + Ok(()) +} + +/// An account holder is never a guest, and that holds after the mint too: a session +/// minted before the account existed ends at the door the moment one does, so an +/// account provisioned in a race with the mint cannot outlive the rule. +#[sqlx::test(fixtures("base"))] +async fn an_account_ends_the_guest_session(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + insert_guest_token(&db, "test-workspace").await?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "guest whoami must resolve"); + + sqlx::query( + "INSERT INTO password (email, password_hash, login_type, super_admin, verified, name) + VALUES ('guest@example.com', 'not-a-real-hash', 'password', false, true, 'Guest')", + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "an account created after the mint ends the guest session at the door" + ); + + Ok(()) +} + +/// An upload goes through an app's `s3_inputs` policy or not at all for a guest: the +/// legacy branch for an app without one uploads with the caller's own standing, which a +/// guest has none of, and an app path with no row must not slip past the confinement. +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn a_guest_cannot_upload_outside_a_policy(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) // no `s3_inputs` + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; + + let upload = |app: &str| { + authed( + client().post(format!( + "{ws}/apps_u/upload_s3_file/{app}?file_key=anything" + )), + GUEST_TOKEN, + ) + .body("x") + .send() + }; + let resp = upload("u/test-user/no_such_app").await?; + assert_eq!( + resp.status(), + 403, + "a path with no app must not escape the guest's confinement: {}", + resp.text().await? + ); + let resp = upload(APP_PATH).await?; + assert_eq!( + resp.status(), + 400, + "without an upload policy a guest is refused like an anonymous caller: {}", + resp.text().await? + ); + + Ok(()) +} + +/// An anonymous app is open to anyone, a guest included, and the guest uses it as +/// itself: the component run and the result read that follows are one identity, so +/// the read's launched-by-me grant matches. Acting as nobody for the run and as the +/// guest for the read would start a job whose result the page can never fetch. +#[sqlx::test(fixtures("base"))] +async fn a_guest_uses_an_anonymous_app_as_itself(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/scripts/create")), ADMIN_TOKEN) + .json(&json!({ + "path": "u/test-user/noop", + "summary": "", + "description": "", + "content": "echo 42", + "language": "bash", + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let anon = "u/test-user/anon_app"; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&app_with_runnable(anon, "anonymous", false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; // scoped to APP_PATH, not `anon` + + let resp = execute(port, "test-workspace", anon, GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let job_id = resp.text().await?; + + let resp = authed( + client().get(format!("{ws}/jobs_u/getupdate/{job_id}")), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "the guest that started the run must be able to read it back: {}", + resp.text().await? + ); + + Ok(()) +} + +/// The embed token a guest mints for a sandboxed app is the one credential handed to +/// untrusted app JS. It must be a guest twice over — resolve like its minter (the +/// label) and be governed like its minter (the sentinel) — or every guest control +/// silently skips the most exposed credential there is. +#[sqlx::test(fixtures("base"))] +async fn a_guest_minted_embed_token_stays_a_guest(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, true)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let secret: String = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + insert_guest_token(&db, "test-workspace").await?; + + // The guest page mints the iframe's token from the guest session. + let resp = authed( + client().get(format!("{ws}/apps_u/embed_token/{secret}")), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a guest must be able to mint: {}", + resp.text().await? + ); + let body: serde_json::Value = resp.json().await?; + let embed = body["token"] + .as_str() + .expect("mint must return a token for an authenticated guest") + .to_string(); + + // Its lifetime is capped at the session that minted it: the requested embed + // validity (12h) is longer than the guest session's (8h in this fixture), and the + // session's expiry is a guest's only revocation. + let parent_exp: chrono::DateTime = + sqlx::query_scalar("SELECT expiration FROM token WHERE token_prefix = 'GUEST_SECR'") + .fetch_one(&db) + .await?; + let child_exp: chrono::DateTime = body["expiration"] + .as_str() + .and_then(|e| e.parse().ok()) + .expect("mint must return the token's expiration"); + assert!( + child_exp <= parent_exp, + "a guest's embed token must not outlive the session that minted it ({child_exp} > {parent_exp})" + ); + + // Resolves — and as a guest, not as the non-member superadmin shape. + let resp = authed(client().get(format!("{ws}/users/whoami")), &embed) + .send() + .await?; + assert_eq!(resp.status(), 200, "the minted token must authenticate"); + let me: serde_json::Value = resp.json().await?; + assert_eq!(me["role"], json!("guest")); + + // Governed: the workspace switch closes it at the door, iframe or not. + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": false })) + .send() + .await?; + let resp = authed(client().get(format!("{ws}/users/whoami")), &embed) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "turning guests off must stop a guest's embed token authenticating" + ); + let resp = execute(port, "test-workspace", APP_PATH, &embed) + .send() + .await?; + assert!( + resp.status().is_client_error() && resp.status() != 404, + "and running components, got {}", + resp.status() + ); + enable_guests(port, "test-workspace").await?; + + // And its scopes are not something the guest's email can later rewrite. The + // guest session itself cannot reach `/users/*` (workspace pin), so model the real + // threat: the same email after promotion, holding an ordinary unpinned session. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label) + VALUES (encode(sha256($1::bytea), 'hex'), 'PROMOTED_S', $2, 'guest@example.com', + 'session')", + ) + .bind(b"PROMOTED_SESSION".as_slice()) + .bind("PROMOTED_SESSION") + .execute(&db) + .await?; + for prefix in [&embed[..10], &GUEST_TOKEN[..10]] { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/users/tokens/update_scopes/{prefix}" + )), + "PROMOTED_SESSION", + ) + .json(&json!({ "scopes": null })) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a promoted account must not be able to rescope its old guest credentials" + ); + } + + Ok(()) +} + +/// The label is the single source of truth: a guest-labelled credential is governed +/// as a guest even if its scopes carry no sentinel. Otherwise every mint that derives +/// a token from a guest session is one forgotten `push` away from an ungoverned +/// non-member credential. +#[sqlx::test(fixtures("base"))] +async fn a_guest_label_is_governed_without_the_sentinel(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let scopes: Vec = guest_scopes() + .into_iter() + .filter(|s| s != "guest") + .collect(); + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'NOSENTINE_', $2, 'guest@example.com', + 'guest_session', $3, 'test-workspace', now() + interval '8 hours')", + ) + .bind(b"NOSENTINEL".as_slice()) + .bind("NOSENTINEL") + .bind(scopes) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{ws}/users/whoami")), "NOSENTINEL") + .send() + .await?; + assert_eq!(resp.status(), 200); + let me: serde_json::Value = resp.json().await?; + assert_eq!( + me["role"], + json!("guest"), + "the label alone must make a credential a guest" + ); + let resp = authed(client().get(format!("{ws}/jobs/list")), "NOSENTINEL") + .send() + .await?; + assert_eq!(resp.status(), 403, "and confine it like one"); + + Ok(()) +} + +/// A guest is someone with no account at all — including a deactivated one. The +/// sign-in path's own account lookup filters on `disabled = false`, so a disabled +/// account reads as absent there; the mint has to refuse on its own or deactivation +/// (manual or SCIM, whose revocation is "delete the tokens") walks straight back in. +#[sqlx::test(fixtures("base"))] +async fn a_disabled_account_cannot_become_a_guest(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + sqlx::query( + "INSERT INTO password (email, password_hash, login_type, super_admin, verified, disabled) + VALUES ('gone@example.com', 'x', 'password', false, true, true)", + ) + .execute(&db) + .await?; + + let mut tx = db.begin().await?; + let cookies = tower_cookies::Cookies::default(); + let minted = windmill_api_users::users::create_guest_session_token( + "gone@example.com", + "test-workspace", + APP_PATH, + &mut tx, + cookies, + ) + .await; + assert!( + matches!(minted, Err(windmill_common::error::Error::NotAuthorized(_))), + "a deactivated account must be refused a guest session, got {minted:?}" + ); + + Ok(()) +} diff --git a/backend/tests/app_guest_jwt_allowance.rs b/backend/tests/app_guest_jwt_allowance.rs new file mode 100644 index 0000000000..85cb51d39e --- /dev/null +++ b/backend/tests/app_guest_jwt_allowance.rs @@ -0,0 +1,128 @@ +//! The guest allowance reached through a guest JWT (`jwt_guest_`). Its own binary +//! because `set_plan` flips a process-global license key, which a test sharing the +//! process could not tolerate (see `app_guest_allowance.rs`). +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::FREE_GUESTS_PER_WINDOW; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +/// Community and Pro are capped, Enterprise is metered. Only a build with both +/// `private` and `enterprise` can meter; every other build is capped whatever this says. +fn set_plan(pro: bool) { + #[cfg(feature = "private")] + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new( + if pro { "test_pro" } else { "" }.to_string(), + )); + let _ = pro; +} + +const JWT_PUB: &str = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzAfqyCh34iYOCW0vg4ejq/zzJlzL\nSZScjnVyPjLGTapEwo4gc6/y1Yudd/v54wKh0OdfTfzAKMPWx/2NWx/ugg==\n-----END PUBLIC KEY-----\n"; +const JWT_PRIV: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgu27S2DbSwUh8BmQb\n/i4/VhNdoXV7PJekhnoceMULYLihRANCAATMB+rIKHfiJg4JbS+Dh6Or/PMmXMtJ\nlJyOdXI+MsZNqkTCjiBzr/LVi513+/njAqHQ519N/MAow9bH/Y1bH+6C\n-----END PRIVATE KEY-----\n"; + +fn guest_jwt(email: &str) -> String { + use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; + let exp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + + 3600; + let claims = json!({ + "email": email, + "workspace_id": "test-workspace", + "app_path": APP_PATH, + "exp": exp, + }); + let jwt = encode( + &Header::new(Algorithm::ES256), + &claims, + &EncodingKey::from_ec_pem(JWT_PRIV.as_bytes()).unwrap(), + ) + .unwrap(); + format!("jwt_guest_{jwt}") +} + +/// A JWT guest is subject to the same allowance as a signed-in one. Past the cap on a +/// capped instance, a stranger's JWT is refused (the auth arm returns 401; the visitor +/// message is only logged, since the arm cannot carry it), while a guest already in the +/// window is let back in. +#[sqlx::test(fixtures("base"))] +async fn a_guest_jwt_is_capped_like_a_signed_in_guest(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({ "public_key": JWT_PUB })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables_v2": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + // The whole allowance, used today (g1..gN). + sqlx::query( + "INSERT INTO guest_activity (email, workspace_id, day) + SELECT 'g' || i || '@example.com', 'test-workspace', CURRENT_DATE + FROM generate_series(1, $1) AS i", + ) + .bind(FREE_GUESTS_PER_WINDOW) + .execute(&db) + .await?; + set_plan(true); + + let resp = authed( + client().get(format!("{ws}/users/whoami")), + &guest_jwt("stranger@example.com"), + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "a stranger's JWT is refused past the cap"); + + let resp = authed( + client().get(format!("{ws}/users/whoami")), + &guest_jwt("g1@example.com"), + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a returning guest's JWT is admitted: {}", + resp.text().await? + ); + + Ok(()) +} diff --git a/backend/tests/app_guest_jwt_entry.rs b/backend/tests/app_guest_jwt_entry.rs new file mode 100644 index 0000000000..7b84130a04 --- /dev/null +++ b/backend/tests/app_guest_jwt_entry.rs @@ -0,0 +1,494 @@ +//! Tests for the guest JWT entry: a guest that enters through a JWT the embedding +//! customer's own backend mints and signs, with no identity-provider round-trip. +//! +//! The key is a per-workspace setting (a PEM public key here), and the token is +//! verified per request against it. A JWT guest is the same identity as a signed-in +//! guest: no `usr` row, no `password` row, no seat, confined to the one app its +//! `app_path` names. These tests pin what a token must carry to be honoured, and the +//! refusals that keep the door narrow: wrong workspace, wrong key, expired, a +//! symmetric algorithm, an email that already has an account, an app not in guest +//! mode, and the workspace switch off. +//! +//! The keys are fixed test vectors (EC P-256, PKCS8), so signing is deterministic and +//! needs no key generation at runtime. + +// Built with these like the sibling guest-execution suite: the guest run executes as +// the publisher through EE on-behalf-of code. CI builds with them. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use std::time::{SystemTime, UNIX_EPOCH}; + +use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; +use serde::Serialize; +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; +const GUEST_EMAIL: &str = "guest@example.com"; + +// A P-256 keypair the workspace verifies against (PUB1), and a second private key +// (PRIV2) that it does not, for the wrong-key refusal. +const PRIV1: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgu27S2DbSwUh8BmQb\n/i4/VhNdoXV7PJekhnoceMULYLihRANCAATMB+rIKHfiJg4JbS+Dh6Or/PMmXMtJ\nlJyOdXI+MsZNqkTCjiBzr/LVi513+/njAqHQ519N/MAow9bH/Y1bH+6C\n-----END PRIVATE KEY-----\n"; +const PUB1: &str = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzAfqyCh34iYOCW0vg4ejq/zzJlzL\nSZScjnVyPjLGTapEwo4gc6/y1Yudd/v54wKh0OdfTfzAKMPWx/2NWx/ugg==\n-----END PUBLIC KEY-----\n"; +const PRIV2: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgjyhWYyI2+z5zTT0B\neI9EuJJ7v0tcNXhvHrq9y2AG1LihRANCAAS40dEdO+tTffhGt4YQv0dStkd6VcWN\n+CHI9QqZAHAJMsNS3Ld+sZe2M6Of0CNR300QJtfp4UIdEVbXBCIxL1D0\n-----END PRIVATE KEY-----\n"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +#[derive(Serialize)] +struct Claims { + email: String, + workspace_id: String, + app_path: String, + exp: u64, + #[serde(skip_serializing_if = "Option::is_none")] + nbf: Option, + #[serde(skip_serializing_if = "Option::is_none")] + iat: Option, +} + +impl Claims { + fn valid() -> Self { + Claims { + email: GUEST_EMAIL.to_string(), + workspace_id: "test-workspace".to_string(), + app_path: APP_PATH.to_string(), + exp: now() + 3600, + nbf: None, + iat: None, + } + } +} + +/// Sign as a bearer (`jwt_guest_`). `priv_pem`/`alg` let a test sign with the +/// wrong key or a refused algorithm. +fn bearer(claims: &Claims, priv_pem: &str, alg: Algorithm) -> String { + let key = match alg { + Algorithm::HS256 => EncodingKey::from_secret(b"a-shared-secret"), + _ => EncodingKey::from_ec_pem(priv_pem.as_bytes()).unwrap(), + }; + let jwt = encode(&Header::new(alg), claims, &key).unwrap(); + format!("jwt_guest_{jwt}") +} + +async fn enable_guests(port: u16, ws: &str, on: bool) -> anyhow::Result<()> { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/workspaces/edit_guest_access" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": on })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +async fn set_guest_jwt_pem(port: u16, ws: &str, pem: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/workspaces/edit_guest_jwt_key" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "public_key": pem })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +fn app(path: &str, execution_mode: &str, sandbox: bool) -> serde_json::Value { + json!({ + "path": path, + "summary": "App", + "value": {}, + "policy": { + "execution_mode": execution_mode, + "sandbox": sandbox, + "triggerables_v2": { + "script/u/test-user/noop": { "static_inputs": {}, "one_of_inputs": {} } + } + } + }) +} + +async fn create_app(port: u16, ws: &str, v: serde_json::Value) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("http://localhost:{port}/api/w/{ws}/apps/create")), + ADMIN_TOKEN, + ) + .json(&v) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + Ok(()) +} + +fn whoami(port: u16, ws: &str, token: &str) -> reqwest::RequestBuilder { + authed( + client().get(format!("http://localhost:{port}/api/w/{ws}/users/whoami")), + token, + ) +} + +/// A valid guest JWT opens its app, runs a component as the publisher, reads the run +/// back, reports `role: guest`, and leaves exactly one `guest_activity` row however +/// many requests it makes. +#[sqlx::test(fixtures("base"))] +async fn a_valid_guest_jwt_opens_its_app(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + let resp = authed( + client().post(format!("http://localhost:{port}/api/w/{ws}/scripts/create")), + ADMIN_TOKEN, + ) + .json(&json!({ + "path": "u/test-user/noop", + "summary": "", + "description": "", + "content": "echo 42", + "language": "bash", + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + + // A distinct email: the activity write is deduplicated by a process-global cache + // keyed on email, workspace and day, and other tests in this binary share the + // guest email, so the count below is only this test's if its email is its own. + let mut claims = Claims::valid(); + claims.email = "activity-guest@example.com".to_string(); + let token = bearer(&claims, PRIV1, Algorithm::ES256); + + let resp = whoami(port, ws, &token).send().await?; + assert_eq!(resp.status(), 200, "guest JWT must authenticate"); + let me: serde_json::Value = resp.json().await?; + assert_eq!(me["role"], json!("guest"), "must read as a guest"); + assert_eq!(me["operator"], json!(true)); + assert_eq!(me["is_admin"], json!(false)); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/apps_u/execute_component/{APP_PATH}" + )), + &token, + ) + .json(&json!({ "component": "a", "path": "script/u/test-user/noop", "args": {} })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let job_id = resp.text().await?; + + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/{ws}/jobs_u/getupdate/{job_id}" + )), + &token, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "the guest that started the run must read it back: {}", + resp.text().await? + ); + + // Several requests, one row: the write is cached per email, workspace and day. + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM guest_activity WHERE email = $1 AND workspace_id = $2 AND jwt_entry", + ) + .bind(&claims.email) + .bind(ws) + .fetch_one(&db) + .await?; + assert_eq!(count, 1, "a JWT guest must leave exactly one activity row"); + + Ok(()) +} + +/// The refusals that keep the door narrow. Each presents a bearer on the workspace's +/// own `whoami`, which the arm reaches only after every gate, so a 401 is the arm +/// saying no rather than a handler. +#[sqlx::test(fixtures("base"))] +async fn guest_jwt_refusals(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + create_app(port, ws, app("u/test-user/members_app", "publisher", false)).await?; + + // Positive control: a token valid against this exact fixture is admitted. Without it a + // broken setup would 401 every bearer below and the whole suite would pass vacuously. + let control = whoami(port, ws, &bearer(&Claims::valid(), PRIV1, Algorithm::ES256)) + .send() + .await?; + assert_eq!(control.status(), 200, "{}", control.text().await?); + + // wrong workspace: the claim must name the route's workspace. + let mut c = Claims::valid(); + c.workspace_id = "other-ws".to_string(); + let wrong_ws = bearer(&c, PRIV1, Algorithm::ES256); + + // wrong key: signed with a key the workspace does not hold. + let wrong_key = bearer(&Claims::valid(), PRIV2, Algorithm::ES256); + + // expired, past the verifier's clock-skew leeway. + let mut c = Claims::valid(); + c.exp = now() - 120; + let expired = bearer(&c, PRIV1, Algorithm::ES256); + + // a symmetric algorithm is never accepted. + let hs256 = bearer(&Claims::valid(), PRIV1, Algorithm::HS256); + + // an email that already has an account is refused, not downgraded. + let mut c = Claims::valid(); + c.email = "test@windmill.dev".to_string(); + let has_account = bearer(&c, PRIV1, Algorithm::ES256); + + // an app not in guest mode. + let mut c = Claims::valid(); + c.app_path = "u/test-user/members_app".to_string(); + let not_guest_app = bearer(&c, PRIV1, Algorithm::ES256); + + // an existing account addressed in a different case still counts as an account: + // the base fixture holds `test@windmill.dev`. + let mut c = Claims::valid(); + c.email = "Test@Windmill.Dev".to_string(); + let mixed_case_account = bearer(&c, PRIV1, Algorithm::ES256); + + // a lifetime past the 24h cap, even with a valid signature. + let mut c = Claims::valid(); + c.exp = now() + 25 * 3600; + let over_lifetime_cap = bearer(&c, PRIV1, Algorithm::ES256); + + // an email with no `@` would become the guest's username and could be read as a + // `u/` or `g/` principal; refused. + let mut c = Claims::valid(); + c.email = "group-admins".to_string(); + let group_shaped_email = bearer(&c, PRIV1, Algorithm::ES256); + + // an email longer than the `guest_activity.email` column: refused before auth, so a + // guest is never admitted without the activity row and audit event the count needs. + let mut c = Claims::valid(); + c.email = format!("{}@example.com", "a".repeat(250)); + let oversized_email = bearer(&c, PRIV1, Algorithm::ES256); + + // an app_path carrying a scope metacharacter would widen the guest's scopes. + let mut c = Claims::valid(); + c.app_path = "u/test-user/*".to_string(); + let wildcard_app_path = bearer(&c, PRIV1, Algorithm::ES256); + + // a valid, signed token past the length cap: without the cap it would deserialize into + // GuestJwtClaims (the extra claim ignored) and verify, so this pins the length check. + let mut payload = serde_json::to_value(Claims::valid()).unwrap(); + payload["padding"] = serde_json::json!("a".repeat(9000)); + let big_jwt = encode( + &Header::new(Algorithm::ES256), + &payload, + &EncodingKey::from_ec_pem(PRIV1.as_bytes()).unwrap(), + ) + .unwrap(); + let oversized_token = format!("jwt_guest_{big_jwt}"); + + // a repeated prefix must not strip down to a valid short token that verifies and is then + // cached under the full bearer key (trim_start_matches would; strip_prefix must not). + let repeated_prefix = format!( + "jwt_guest_{}", + bearer(&Claims::valid(), PRIV1, Algorithm::ES256) + ); + + for (label, token) in [ + ("wrong workspace", wrong_ws), + ("wrong key", wrong_key), + ("expired", expired), + ("HS256", hs256), + ("email with an account", has_account), + ("app not in guest mode", not_guest_app), + ("mixed-case account", mixed_case_account), + ("over the 24h lifetime cap", over_lifetime_cap), + ("group-shaped email", group_shaped_email), + ("oversized email", oversized_email), + ("wildcard app_path", wildcard_app_path), + ("oversized token", oversized_token), + ("repeated prefix", repeated_prefix), + ] { + let resp = whoami(port, ws, &token).send().await?; + assert_eq!(resp.status(), 401, "{label} must be refused"); + } + + Ok(()) +} + +/// The workspace switch gates a JWT guest exactly as it gates a signed-in one, at the +/// auth door, so turning guests off closes the JWT entry too. +#[sqlx::test(fixtures("base"))] +async fn guest_jwt_needs_the_workspace_switch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + let token = bearer(&Claims::valid(), PRIV1, Algorithm::ES256); + + // Switch off (the default): refused. + let resp = whoami(port, ws, &token).send().await?; + assert_eq!( + resp.status(), + 401, + "a JWT guest must be refused while guests are off" + ); + + // Switch on: through. + enable_guests(port, ws, true).await?; + let resp = whoami(port, ws, &token).send().await?; + assert_eq!( + resp.status(), + 200, + "with guests on, the JWT guest is admitted" + ); + + // Off again: closed on the next request. + enable_guests(port, ws, false).await?; + let resp = whoami(port, ws, &token).send().await?; + assert_eq!( + resp.status(), + 401, + "turning guests off closes the JWT guest again" + ); + + Ok(()) +} + +/// A guest JWT is pinned to the workspace its claim names, so it authenticates on no +/// workspace-less route: the arm has no workspace to check the claim against. +#[sqlx::test(fixtures("base"))] +async fn guest_jwt_rejected_on_workspaceless_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + let token = bearer(&Claims::valid(), PRIV1, Algorithm::ES256); + + let resp = authed( + client().get(format!("http://localhost:{port}/api/users/tokens/list")), + &token, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest JWT must not authenticate on a workspace-less route" + ); + + Ok(()) +} + +/// An embed token a JWT guest mints for a sandboxed app is capped at the JWT's own +/// expiry: a JWT has no token row, so the cap is carried through the auth cache. It +/// must not outlive the JWT, which is the guest's only revocation. +#[sqlx::test(fixtures("base"))] +async fn a_guest_jwt_derived_embed_token_is_capped(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", true)).await?; + let secret: String = authed( + client().get(format!( + "http://localhost:{port}/api/w/{ws}/apps/secret_of/{APP_PATH}" + )), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + + let claims = Claims::valid(); + let jwt_exp = claims.exp; + let token = bearer(&claims, PRIV1, Algorithm::ES256); + + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/{ws}/apps_u/embed_token/{secret}" + )), + &token, + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let body: serde_json::Value = resp.json().await?; + let child_exp: chrono::DateTime = body["expiration"] + .as_str() + .and_then(|e| e.parse().ok()) + .expect("mint must return the token's expiration"); + assert!( + child_exp.timestamp() as u64 <= jwt_exp, + "the derived embed token ({child_exp}) must not outlive the JWT (exp {jwt_exp})" + ); + + // And it resolves as a guest. + let embed = body["token"].as_str().expect("mint must return a token"); + let resp = whoami(port, ws, embed).send().await?; + assert_eq!(resp.status(), 200); + let me: serde_json::Value = resp.json().await?; + assert_eq!(me["role"], json!("guest")); + + Ok(()) +} + +/// A workspace with no guest key of its own falls back to the instance issuer +/// (`JWT_EXT_JWKS_URL`), so an operator running one issuer configures it once. Verified as a +/// guest here in CE; a full login from that issuer stays EE (`jwt_ext_`). +#[sqlx::test(fixtures("base"))] +async fn no_workspace_key_falls_back_to_the_instance_issuer( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::guest_jwt::{key_source, GuestJwtKeySource}; + let url = "https://issuer.example.com/jwks.json"; + unsafe { std::env::set_var("JWT_EXT_JWKS_URL", url) }; + let src = key_source(&db, "test-workspace").await; + unsafe { std::env::remove_var("JWT_EXT_JWKS_URL") }; + assert!( + matches!(src?, Some(GuestJwtKeySource::JwksUrl(u)) if u == url), + "no workspace key falls back to the instance issuer" + ); + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index fc766253bc..590963b291 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -939,6 +939,127 @@ export function main() { return midValue(); }"#, Ok(()) } +/// A run with local modules and no lock executes the bundle its lock generation built, which +/// kept the imported script's pin; the run must still load the one copy in node_modules, and +/// leave the script's own data alone even where it matches the pinned specifier. +#[sqlx::test(fixtures("base"))] +async fn test_bun_modules_run_loads_imported_pin_from_node_modules( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + insert_deployed_bun_script( + &db, + "f/pinned_import_modules/module", + 41240002, + r#"import * as isNumber from "is-number@6.0.0"; +export const ns = isNumber; +export const label = "is-number@6.0.0";"#, + ) + .await; + + let job = JobPayload::Code(RawCode { + content: r#"import * as isNumber from "is-number"; +import { ns, label } from "/f/pinned_import_modules/module"; +import { local } from "./helper"; +export function main() { return [ns === isNumber, label, local()]; }"# + .into(), + path: Some("f/pinned_import_modules/main".into()), + language: ScriptLang::Bun, + modules: Some(std::collections::HashMap::from([( + "helper.ts".to_string(), + windmill_common::scripts::ScriptModule { + content: "export const local = () => 'local';".into(), + language: ScriptLang::Bun, + lock: None, + }, + )])), + ..RawCode::default() + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + assert_eq!( + result, + serde_json::json!([true, "is-number@6.0.0", "local"]) + ); + Ok(()) +} + +async fn bun_dependency_lock(db: &Pool, port: u16, path: &str, content: &str) -> String { + let deps = RunJob::from(JobPayload::RawScriptDependencies { + script_path: path.into(), + content: content.into(), + language: ScriptLang::Bun, + }) + .run_until_complete(db, false, port) + .await + .json_result() + .unwrap(); + let Some(lock) = deps["lock"].as_str() else { + panic!("the dependency job returned no lock: {deps}"); + }; + lock.to_string() +} + +/// Bundling a locked script resolves a pinned dynamic `import()` as written. Where that fails, both +/// the dependency job and a run that finds no cached bundle must still build it, from the version +/// the lock pins; where bun tolerates the failure, the bundle must stay as written. +#[sqlx::test(fixtures("base"))] +async fn test_bun_bundles_pinned_dynamic_import(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The dependency job saves the script's bundle here; the server binary creates it at startup. + std::fs::create_dir_all(&*windmill_worker::BUN_BUNDLE_CACHE_DIR)?; + + const PATH: &str = "f/pinned_dynamic_import/main"; + // Every `script` call draws its own nonce, so each job below misses every bundle cached before + // it, this test's included, and has to build one: a cached bundle skips the build under test. + // 4.17.20 is not npm's `latest`, so a bundle that lost the pin cannot match by accident. + let script = |body: &str| { + format!( + "export async function main() {{\n {body}\n}}\n// {}", + Uuid::new_v4() + ) + }; + let import = r#"const m = await import("lodash@4.17.20"); return (m.default ?? m).VERSION;"#; + + let lock = bun_dependency_lock(&db, port, PATH, &script(import)).await; + let result = RunJob::from(JobPayload::Code(RawCode { + content: script(import), + path: Some(PATH.into()), + language: ScriptLang::Bun, + lock: Some(lock), + ..RawCode::default() + })) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + assert_eq!(result, serde_json::json!("4.17.20")); + + let tolerated = script(&format!("try {{ {import} }} catch {{ return null; }}")); + let lock = bun_dependency_lock(&db, port, PATH, &tolerated).await; + let (bundle, _) = windmill_worker::compute_bundle_local_and_remote_path( + &tolerated, + &lock, + PATH, + Some(&db), + "test-workspace", + &None, + None, + ) + .await; + assert!(std::fs::read_to_string(bundle)?.contains("lodash@4.17.20")); + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/tests/fixtures/git_sync_autopull_recovery.sql b/backend/tests/fixtures/git_sync_autopull_recovery.sql new file mode 100644 index 0000000000..6392da50ab --- /dev/null +++ b/backend/tests/fixtures/git_sync_autopull_recovery.sql @@ -0,0 +1,10 @@ +-- A workspace whose auto-pulled repository last recorded a head-check failure while +-- already synced to head "aaa": the state a recovery write is decided on. + +INSERT INTO workspace (id, name, owner) VALUES ('ap-ws', 'ap-ws', 'test-user'); + +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('ap-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "auto_pull":{"enabled":true,"mode":"polling", + "last_synced_sha":{"main":"aaa"}, + "last_pull_status":{"success":false,"at":1,"error":"head check failed: x"}}}]}'); diff --git a/backend/tests/fixtures/git_sync_fork_credential.sql b/backend/tests/fixtures/git_sync_fork_credential.sql new file mode 100644 index 0000000000..1c1d8bd1e1 --- /dev/null +++ b/backend/tests/fixtures/git_sync_fork_credential.sql @@ -0,0 +1,55 @@ +-- A parent whose git-sync repository has a recorded credential, and the workspace +-- shapes the credential lookup and the qualification predicate have to tell apart. +-- +-- The credential itself is shared down the fork chain; the recorded *status* is +-- not, because it describes one repository and a fork can repoint its copy of the +-- resource. Forks get a status by fork creation copying it, which no fixture here +-- simulates, so a fork without one is a workspace nothing has checked yet. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('parent-ws', 'parent-ws', 'test-user', NULL), + ('fork-ws', 'fork-ws', 'test-user', 'parent-ws'), + -- A fork of a fork: the shape a fork of a dev workspace takes, and the one a + -- parent-only lookup misses. + ('deep-fork-ws', 'deep-fork-ws', 'test-user', 'fork-ws'), + ('errored-fork-ws', 'errored-fork-ws', 'test-user', 'parent-ws'), + ('orphan-ws', 'orphan-ws', 'test-user', NULL); + +-- A stored credential is encrypted with its own workspace's key. +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('parent-ws', 'cloud', 'parent-key'), + ('fork-ws', 'cloud', 'fork-key'), + ('deep-fork-ws', 'cloud', 'deep-fork-key'), + ('errored-fork-ws', 'cloud', 'errored-fork-key'), + ('orphan-ws', 'cloud', 'orphan-key'); + +-- The parent holds the credential. +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('parent-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":true,"checked_at":1788500000}}]}'), + +-- A fork inherits the repository but not the credential: this is what +-- clone_workspace_data leaves behind. + ('fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- Two levels down, so neither the credential nor its status is one hop away. + ('deep-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- A fork whose own credential has since failed. Its own standing must win over +-- the parent's healthy record rather than being papered over. + ('errored-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":false,"checked_at":1788500000, + "error":"GitLab no longer accepts this token"}}]}'), + +-- No credential and no parent to borrow one from. + ('orphan-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'); + +-- The resource each repository entry names, all pointing at the same repository. +-- The errored fork carries its token in the URL, the way a repository configured +-- by hand does: a plain remote, whatever the chain above it holds. +INSERT INTO resource (workspace_id, path, value, resource_type) VALUES + ('parent-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('deep-fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('errored-fork-ws', 'u/admin/repo', '{"url":"https://oauth2:glpat-inline@gitlab.com/grp/proj.git"}', 'git_repository'), + ('orphan-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'); diff --git a/backend/tests/folder_default_permissioned_as.rs b/backend/tests/folder_default_permissioned_as.rs index 7aa4f0a003..238d7cdb32 100644 --- a/backend/tests/folder_default_permissioned_as.rs +++ b/backend/tests/folder_default_permissioned_as.rs @@ -630,7 +630,7 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu ); assert_eq!( policy["on_behalf_of_email"], "group-wm_deployers@windmill.dev", - "app policy.on_behalf_of_email gets folder default email" + "the stored address is derived from that principal" ); // 7b. Admin, non-matching path — acting user diff --git a/backend/tests/git_sync_autopull_recovery.rs b/backend/tests/git_sync_autopull_recovery.rs new file mode 100644 index 0000000000..d3ba8b7c9f --- /dev/null +++ b/backend/tests/git_sync_autopull_recovery.rs @@ -0,0 +1,143 @@ +//! A recorded auto-pull failure is cleared once the tracked head is observed again +//! at the already-synced sha, and only then: the decision is taken on a snapshot, +//! so the write must re-check the stored row rather than overwrite it. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use std::collections::HashMap; +use uuid::Uuid; +use windmill_common::workspaces::AutoPullStatus; +use windmill_git_sync::{clear_auto_pull_failure, persist_auto_pull_state}; + +const WS: &str = "ap-ws"; +const REPO: &str = "$res:u/admin/repo"; + +/// The failure the fixture records, as the poller would have read it. +fn fixture_failure() -> AutoPullStatus { + AutoPullStatus { + synced_sha: None, + at: 1, + job_id: None, + success: false, + error: Some("head check failed: x".to_string()), + } +} + +fn recovered(head: &str) -> AutoPullStatus { + AutoPullStatus { + synced_sha: Some(head.to_string()), + at: 2, + job_id: None, + success: true, + error: None, + } +} + +async fn stored_auto_pull(db: &Pool) -> anyhow::Result { + let git_sync: serde_json::Value = + sqlx::query_scalar("SELECT git_sync FROM workspace_settings WHERE workspace_id = $1") + .bind(WS) + .fetch_one(db) + .await?; + Ok(git_sync["repositories"][0]["auto_pull"].clone()) +} + +/// The recovery every test below runs, decided on the fixture's failure at head +/// "aaa": live in the first test, stale in the two that move the stored state +/// first. +async fn recovery_for_the_fixture_failure(db: &Pool) -> anyhow::Result<()> { + clear_auto_pull_failure( + db, + WS, + REPO, + "main", + "aaa", + &fixture_failure(), + &recovered("aaa"), + ) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn recovery_clears_the_failure_at_the_synced_head(db: Pool) -> anyhow::Result<()> { + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_pull_status"]["success"], true); + assert!(auto_pull["last_pull_status"].get("error").is_none()); + assert_eq!(auto_pull["last_pull_status"]["synced_sha"], "aaa"); + assert_eq!( + auto_pull["last_synced_sha"]["main"], "aaa", + "the sha map is not part of a recovery write" + ); + Ok(()) +} + +/// Between the poller observing head "aaa" unchanged and its recovery write, a +/// webhook may have enqueued newer head "bbb". The stale recovery must leave that +/// optimistic state (sha, success, job id) in place; the job's completion hook +/// relies on it, and rolling the sha back would re-enqueue "bbb" on the next tick. +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn stale_recovery_leaves_a_newer_state_alone(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + let advanced = AutoPullStatus { + synced_sha: Some("bbb".to_string()), + at: 3, + job_id: Some(job_id), + success: true, + error: None, + }; + persist_auto_pull_state( + &db, + WS, + REPO, + &HashMap::from([("main".to_string(), "bbb".to_string())]), + &advanced, + ) + .await?; + + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_synced_sha"]["main"], "bbb"); + assert_eq!(auto_pull["last_pull_status"]["synced_sha"], "bbb"); + assert_eq!(auto_pull["last_pull_status"]["job_id"], job_id.to_string()); + assert_eq!(auto_pull["last_pull_status"]["at"], 3); + Ok(()) +} + +/// The head can stay at "aaa" while a newer failure is recorded (a later head +/// check, a pull job that failed). A recovery decided on the older failure must +/// not paper over the newer one, whether it differs by timestamp or, within the +/// same second, only by its error. +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn stale_recovery_keeps_a_newer_failure_at_the_same_head( + db: Pool, +) -> anyhow::Result<()> { + let same_sha = HashMap::from([("main".to_string(), "aaa".to_string())]); + for newer in [ + AutoPullStatus { + at: 5, + error: Some("head check failed: later".to_string()), + ..fixture_failure() + }, + AutoPullStatus { + error: Some("head check failed: same second".to_string()), + ..fixture_failure() + }, + ] { + persist_auto_pull_state(&db, WS, REPO, &same_sha, &newer).await?; + + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_pull_status"]["success"], false); + assert_eq!(auto_pull["last_pull_status"]["at"], newer.at); + assert_eq!( + auto_pull["last_pull_status"]["error"], + newer.error.as_deref().unwrap() + ); + } + Ok(()) +} diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs new file mode 100644 index 0000000000..10cf1e3735 --- /dev/null +++ b/backend/tests/git_sync_fork_credential.rs @@ -0,0 +1,280 @@ +//! A fork reaches the git credential held above it in its fork chain. +//! +//! Fork creation copies the parent's git-sync repositories but not the credential, +//! which is stored per workspace so that rotation has one owner. Chains nest (a +//! fork of a dev workspace, a fork of that), so the depth-2 cases here are what +//! keep the lookup from regressing to the parent. +//! +//! The recorded *status* is not shared the same way: it describes one repository, +//! and a fork can repoint its copy of the resource, so each workspace answers from +//! its own record and gets one by fork creation copying it down. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::git_sync_ee::{ + create_repo_webhook, git_credential_for_url, repo_provider, repo_supports_managed_git_features, + set_git_credential, GitProvider, +}; +use windmill_common::workspaces::GitCredentialProvider; + +const REPO: &str = "$res:u/admin/repo"; +const URL: &str = "https://gitlab.com/grp/proj.git"; + +/// A repository is managed when a credential is held for the repository its +/// URL names now and the last check found it healthy. The recorded status is +/// keyed by resource path, so alone it would outlive a repoint; the held +/// credential alone says nothing about whether the host still accepts it. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn credential_status_is_a_workspaces_own(db: Pool) -> anyhow::Result<()> { + assert!( + !repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "a healthy status with nothing held behind it does not qualify" + ); + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert!( + repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "the workspace holding both the credential and the recorded status qualifies" + ); + assert!( + !repo_supports_managed_git_features(&db, "fork-ws", REPO).await, + "a fork borrowing the credential with no record of its own does not: the \ + status describes one repository, and this fork's resource could name another" + ); + assert!( + !repo_supports_managed_git_features(&db, "errored-fork-ws", REPO).await, + "a workspace whose own credential failed stays disqualified" + ); + Ok(()) +} + +/// The host a repository talks to is declared when its credential is stored, and +/// travels with the credential down the fork chain. +/// +/// Read from the recorded status instead, a fork answered with the default +/// provider until its own check ran, which is long enough to register a webhook +/// against the wrong receiver. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn the_provider_comes_from_the_credential_and_reaches_forks( + db: Pool, +) -> anyhow::Result<()> { + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitHub, + "with nothing stored there is no declaration to read, so the default stands" + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitLab, + "the workspace that stored it reads its own declaration" + ); + assert_eq!( + repo_provider(&db, "fork-ws", REPO).await, + GitProvider::GitLab, + "and a fork resolving that credential reads it too, without a check of its own" + ); + assert_eq!( + repo_provider(&db, "deep-fork-ws", REPO).await, + GitProvider::GitLab, + "two levels down as well" + ); + assert_eq!( + repo_provider(&db, "orphan-ws", REPO).await, + GitProvider::GitHub, + "a workspace outside the chain resolves no credential and no declaration" + ); + assert_eq!( + repo_provider(&db, "errored-fork-ws", REPO).await, + GitProvider::GitHub, + "a token written into the URL makes the repository a plain remote: the \ + parent's credential is not consulted and no host is declared" + ); + Ok(()) +} + +/// The stored credential is shared with forks and keyed by one repository. +/// +/// Both properties are the point of keeping it in `workspace_settings` under the +/// repository's identity: sharing is what stops a rotation from stranding every +/// fork on a revoked token, and the key is what stops a rewritten resource URL +/// from carrying the token to a host of the writer's choosing. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_fork_reads_an_ancestors_credential_for_the_bound_repository_only( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "the workspace that stored it reads it back" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork stores none of its own and resolves the parent's" + ); + assert_eq!( + git_credential_for_url(&db, "deep-fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork of a fork resolves the root's, two levels up" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", "https://evil.example/grp/proj.git").await?, + None, + "a resource repointed at another repository asks for that one's \ + credential and finds none" + ); + assert_eq!( + git_credential_for_url(&db, "orphan-ws", URL).await?, + None, + "a workspace with no credential and no parent resolves nothing" + ); + Ok(()) +} + +/// One repository's credential is untouched by another's. +/// +/// The key is the repository, so picking a second repository stores beside the +/// first rather than over it. Keyed by the resource instead, a workspace editing +/// one repository's resource to point somewhere else would replace the token the +/// original repository was still syncing with. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn each_repository_keeps_its_own_credential(db: Pool) -> anyhow::Result<()> { + const OTHER_URL: &str = "https://gitlab.com/grp/other.git"; + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-first", + GitCredentialProvider::Gitlab, + ) + .await?; + set_git_credential( + &db, + "parent-ws", + OTHER_URL, + "glpat-second", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-first"), + "storing a second repository's token leaves the first's in place" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second") + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-replacement", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-replacement"), + "storing the same repository again replaces rather than duplicates" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second"), + "and still leaves the other repository alone" + ); + Ok(()) +} + +/// A credential issued for `https` is not served for the `http` spelling. +/// +/// The resource holding the URL is writable by anyone with write on its path, so +/// without the scheme in the key that edit would send the token over cleartext. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_credential_is_not_served_over_a_downgraded_transport( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", "http://gitlab.com/grp/proj.git").await?, + None + ); + Ok(()) +} + +/// A GitLab the server cannot reach is the error reported, not the GitHub App +/// lookup that runs after it: for a self-managed GitLab behind a firewall or an +/// untrusted certificate, "no GitHub App installation" names neither the host +/// nor the cause. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn an_unreachable_gitlab_host_is_the_reported_error( + db: Pool, +) -> anyhow::Result<()> { + let err = create_repo_webhook( + &db, + "parent-ws", + "http://glpat-secret@127.0.0.1:1/grp/proj.git", + "https://windmill.example/api/w/parent-ws/git_sync/webhook/gitlab", + "hook-secret", + ) + .await + .expect_err("nothing listens on port 1"); + assert!( + err.to_string().contains("Could not reach the git host"), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("glpat-secret"), + "the URL credential leaked into the error: {err}" + ); + Ok(()) +} diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index ddd94ea079..64d7854173 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -1442,7 +1442,10 @@ async fn declarative_sync_rejects_an_unusable_webhook_base_url(db: Pool) { + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + let mut desired = BTreeMap::new(); + desired.insert( + "base_url".to_string(), + serde_json::json!("https://wm.example.com"), + ); + desired.insert( + "instance_banner".to_string(), + serde_json::json!({ "enabled": true, "message": "down", "link": "javascript:alert(1)" }), + ); + + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err("a javascript: banner link must fail the sync"); + assert!( + err.to_string().contains("instance_banner"), + "the error should name the offending setting, got: {err}" + ); + + assert_eq!( + count_global_settings(&db).await, + before, + "validation must run before anything is applied" + ); + assert!( + get_global_setting(&db, "base_url").await.is_none(), + "the other settings in the same apply must not have been written either" + ); +} + +#[sqlx::test(fixtures("base"))] +async fn declarative_sync_rejects_an_unusable_default_allowed_origins(db: Pool) { + // The declarative writers (the sync-config CLI, the operator's ConfigMap + // sync) do not run the HTTP layer's pre-write hook, so an origin list that + // cannot be parsed would persist here, be dropped at boot, and leave the + // instance with no restriction at all. + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + for bad in [ + serde_json::json!([""]), + serde_json::json!(["https://a.example,https://b.example"]), + serde_json::json!("null"), + ] { + let mut desired = BTreeMap::new(); + desired.insert( + "http_route_default_allowed_origins".to_string(), + bad.clone(), + ); + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err(&format!("{bad} must fail the sync")); + assert!( + err.to_string() + .contains("http_route_default_allowed_origins"), + "the error should name the offending setting, got: {err}" + ); + } + + assert_eq!( + count_global_settings(&db).await, + before, + "a rejected sync must not have persisted anything" + ); + + // A usable list still syncs. + let mut desired = BTreeMap::new(); + desired.insert( + "http_route_default_allowed_origins".to_string(), + serde_json::json!(["https://app.example.com"]), + ); + windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect("a valid origin list must sync"); +} diff --git a/backend/tests/jobs_cross_site_get.rs b/backend/tests/jobs_cross_site_get.rs new file mode 100644 index 0000000000..20aa09a0a5 --- /dev/null +++ b/backend/tests/jobs_cross_site_get.rs @@ -0,0 +1,138 @@ +//! Regression test for cross-site GET CSRF on the job-run endpoints that can run a Hub script. +//! +//! `run_wait_result/p/{path}` and `run_and_stream/p/{path}` answer GET and, for a `hub/` path, +//! run any public Hub script. The session cookie is `SameSite=Lax`, so a browser attaches it +//! to a cross-site top-level GET navigation, and an argument written `$var:` or +//! `$res:` is resolved as the caller: an attacker page could make a logged-in browser +//! run a generic Hub script and hand it the victim's secrets. CORS hides the response but not +//! the side effect. +//! +//! `CrossSiteGetGuard` refuses such a request. Workspace scripts are deliberately not refused, +//! since they only run code the workspace's own members deployed. A request carrying its own +//! credential is allowed; the one case that regresses is a signed-in user clicking a `?token=` +//! Hub-script link from another site, because `extract_token` gives the cookie precedence and +//! exempting the parameter would let `?token=junk` reinstate the vector. +//! +//! This test pins down: +//! - both endpoints refuse a cross-site cookie GET to a Hub script (the core fix), whether +//! `Sec-Fetch-Site` says so or, with no such header (plain http), a cross-host `Referer`, +//! - a junk `token` query parameter does not buy a pass, +//! - the scope: the same request to a workspace script is not refused, +//! - a Hub-script request with its own credential (bearer, or `?token=` and no cookie), or +//! sent as a POST, gets through. +//! +//! No runnable exists and no Hub is contacted. A request that gets past the guard fails as +//! not-found on a workspace path, and on the non-numeric version in `hub/x/...` for a Hub +//! path, which is rejected while resolving the runnable, before any call to the Hub. + +use reqwest::StatusCode; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const HUB_GETS: [&str; 2] = [ + "run_wait_result/p/hub/x/absent", + "run_and_stream/p/hub/x/absent", +]; +const WORKSPACE_GETS: [&str; 2] = [ + "run_wait_result/p/u/test-user/absent", + "run_and_stream/p/u/test-user/absent", +]; + +async fn send(req: reqwest::RequestBuilder) -> anyhow::Result<(StatusCode, String)> { + let resp = req.send().await?; + let status = resp.status(); + Ok((status, resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn test_cross_site_get_cannot_run_hub_scripts(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/jobs", + server.addr.port() + ); + let client = reqwest::Client::new(); + let cookie_get = |path: &str| { + client + .get(format!("{base}/{path}")) + .header("Cookie", "token=SECRET_TOKEN") + }; + + // ---- CORE REGRESSION: a cross-site cookie GET cannot run a Hub script. + for path in HUB_GETS { + let refused = [ + ( + "Sec-Fetch-Site: cross-site", + cookie_get(path).header("Sec-Fetch-Site", "cross-site"), + ), + // Plain http gets no `Sec-Fetch-*` at all, so `Referer` is the only signal left. + ( + "cross-host Referer with no Sec-Fetch-Site", + cookie_get(path).header("Referer", "http://attacker.example/page"), + ), + // The cookie outranks a `token` query parameter when authenticating. + ( + "junk ?token= next to the cookie", + client + .get(format!("{base}/{path}?token=junk")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ]; + for (name, req) in refused { + let (status, body) = send(req).await?; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "{path} [{name}] must be refused: {body}" + ); + } + } + + // ---- Scope: the same request to a workspace script is not refused. + for path in WORKSPACE_GETS { + let (status, body) = send(cookie_get(path).header("Sec-Fetch-Site", "cross-site")).await?; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{path} is a workspace script and must reach the handler: {body}" + ); + } + + // ---- A Hub-script request that carries its own credential, or is a POST, gets through. + let hub = HUB_GETS[1]; + let allowed = [ + ( + "cross-origin bearer token", + client + .get(format!("{base}/{hub}")) + .header("Authorization", "Bearer SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "cross-origin ?token= with no cookie", + client + .get(format!("{base}/{hub}?token=SECRET_TOKEN")) + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "POST with the cookie", + client + .post(format!("{base}/{hub}")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site") + .json(&serde_json::json!({})), + ), + ]; + for (name, req) in allowed { + let (status, body) = send(req).await?; + assert!( + body.contains("Invalid hub script version"), + "{name} must get past the guard to runnable resolution (got {status}): {body}" + ); + } + + Ok(()) +} diff --git a/backend/tests/nativets_dedicated.rs b/backend/tests/nativets_dedicated.rs index 1c6987a480..1e9aef3228 100644 --- a/backend/tests/nativets_dedicated.rs +++ b/backend/tests/nativets_dedicated.rs @@ -12,7 +12,7 @@ mod prewarmed_isolate_tests { use windmill_worker::{build_loader, LoaderMode, BUN_PATH}; fn default_annotation() -> NativeAnnotation { - NativeAnnotation { useragent: None, proxy: None } + NativeAnnotation::default() } /// Bundle a TypeScript script into JS suitable for `PrewarmedIsolate`. diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index 3a81f31021..1a4040d390 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -627,3 +627,121 @@ async fn test_root_job_span_relocated_to_inbound_trace() { expected_uuid_trace ); } + +// ═══════════════════════════════════════════════════════════════════════ +// RESOURCE ATTRIBUTES (OTEL_RESOURCE_ATTRIBUTES) +// ═══════════════════════════════════════════════════════════════════════ + +fn resource_attrs() -> std::collections::HashMap { + otlp_service_resource( + &windmill_common::utils::Mode::Worker, + "fallback-host", + "dev", + ) + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_merges_env_attributes_without_losing_windmill_identity() { + // These take precedence over the hostname argument and over OTEL_RESOURCE_ATTRIBUTES, + // so clear them or an ambient one fails the assertions below for an unrelated reason. + for var in [ + "OTEL_HOST_NAME", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + ] { + std::env::remove_var(var); + } + std::env::set_var( + "OTEL_RESOURCE_ATTRIBUTES", + "k8s.pod.uid=abc-123,service.name=injected,host.name=injected", + ); + let attrs = resource_attrs(); + std::env::remove_var("OTEL_RESOURCE_ATTRIBUTES"); + + // Attributes the deployment injects reach the exporters. + assert_eq!( + attrs.get("k8s.pod.uid").map(String::as_str), + Some("abc-123") + ); + // OTEL_RESOURCE_ATTRIBUTES is the secondary resource, so Windmill's own values still win. + assert_eq!( + attrs.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + attrs.get("host.name").map(String::as_str), + Some("fallback-host") + ); +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_dedicated_overrides_win() { + // A deployment sets these per pod, e.g. from Kubernetes downward-API labels. The + // competing service.name must lose: the spec ranks OTEL_SERVICE_NAME above it. + std::env::set_var("OTEL_RESOURCE_ATTRIBUTES", "service.name=should-lose"); + std::env::set_var("OTEL_SERVICE_NAME", "windmill-workers"); + std::env::set_var("OTEL_SERVICE_VERSION", "1.802.0"); + std::env::set_var("OTEL_HOST_NAME", "pod-7"); + let overridden = resource_attrs(); + + // An empty value means unset, which is what the downward API yields for a missing label. + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + ] { + std::env::set_var(var, ""); + } + let empty = resource_attrs(); + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_RESOURCE_ATTRIBUTES", + ] { + std::env::remove_var(var); + } + let unset = resource_attrs(); + + assert_eq!( + overridden.get("service.name").map(String::as_str), + Some("windmill-workers") + ); + assert_eq!( + overridden.get("service.version").map(String::as_str), + Some("1.802.0") + ); + assert_eq!( + overridden.get("host.name").map(String::as_str), + Some("pod-7") + ); + + assert_eq!( + empty.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + empty.get("host.name").map(String::as_str), + Some("fallback-host") + ); + assert_eq!( + empty.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); + + // With nothing set at all — the default deployment — SdkProvidedResourceDetector still + // contributes service.name = "unknown_service". Ours has to overwrite it. + assert_eq!( + unset.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + unset.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); +} diff --git a/backend/tests/postgres_trigger_scope.rs b/backend/tests/postgres_trigger_scope.rs index e8ba36ebb9..1080138eb0 100644 --- a/backend/tests/postgres_trigger_scope.rs +++ b/backend/tests/postgres_trigger_scope.rs @@ -25,6 +25,7 @@ fn scoped_authed(scopes: Vec<&str>) -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } diff --git a/backend/tests/preserve_on_behalf_of.rs b/backend/tests/preserve_on_behalf_of.rs index 2a1f3895ca..0c8c0b76a3 100644 --- a/backend/tests/preserve_on_behalf_of.rs +++ b/backend/tests/preserve_on_behalf_of.rs @@ -405,11 +405,12 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { // 7. App: Admin preserves on_behalf_of // ======================================== + // Principal only, so the stored address can only have come from deriving it. let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") .json(&new_app_with_on_behalf_of( "u/test-user/app_admin_preserve", Some("u/original-user"), - Some("original@windmill.dev"), + None, true, )) .send() @@ -434,10 +435,24 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/original-user"), "Admin should preserve app on_behalf_of" ); + // The address is written through from the principal, never taken from the request, so the + // stored copy can only agree with it. assert_eq!( policy.get("on_behalf_of_email").and_then(|v| v.as_str()), Some("original@windmill.dev"), - "Admin should preserve app on_behalf_of_email" + "the stored address is derived from the principal, not the one the client sent" + ); + let resp = authed( + client().get(format!("{base}/apps/get/p/u/test-user/app_admin_preserve")), + "SECRET_TOKEN", + ) + .send() + .await?; + let returned: serde_json::Value = resp.json().await?; + assert_eq!( + returned["policy"]["on_behalf_of_email"].as_str(), + Some("original@windmill.dev"), + "the response returns the address written through from the principal" ); // ======================================== @@ -476,11 +491,6 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/original-user"), "Deployer should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Deployer should preserve app on_behalf_of_email" - ); // ======================================== // 9. App: Non-admin cannot preserve @@ -518,10 +528,26 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/test-user-2"), "Non-admin should have their own permissioned_as as app on_behalf_of" ); + + // ======================================== + // 9b. App: a policy naming two different accounts is rejected + // ======================================== + + // This is the shape a workspace deploy produces when it carries the source + // workspace's principal beside the target's address. + let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") + .json(&new_app_with_on_behalf_of( + "u/test-user/app_mismatched_pair", + Some("u/original-user"), + Some("test2@windmill.dev"), + true, + )) + .send() + .await?; assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("test2@windmill.dev"), - "Non-admin should have their own email as app on_behalf_of_email" + resp.status(), + 400, + "a policy whose two halves name different accounts must be rejected" ); // ======================================== @@ -1238,11 +1264,6 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/original-user"), "Admin update should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Admin update should preserve app on_behalf_of_email" - ); // ======================================== // Deployer updates with preserve flag @@ -1304,11 +1325,6 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/original-user"), "Deployer update should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Deployer update should preserve app on_behalf_of_email" - ); // ======================================== // Non-admin cannot preserve on update @@ -1370,10 +1386,62 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/test-user-2"), "Non-admin update should overwrite app on_behalf_of with their own" ); + + Ok(()) +} + +/// A superadmin acting outside their workspaces has no `usr` row, so the per-workspace rename +/// sweep never reaches the apps that name them. Their principal is their instance username, so +/// without a global sweep a rename leaves those apps naming an account that resolves to nobody. +#[sqlx::test(fixtures("preserve_on_behalf_of"))] +async fn test_rename_sweeps_external_superadmin_app_identity( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + let path = "u/original-user/app_run_by_external_superadmin"; + + let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") + .json(&new_app_with_on_behalf_of( + path, + Some("u/superadmin-external"), + Some("superadmin-external@windmill.dev"), + true, + )) + .send() + .await?; assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("test2@windmill.dev"), - "Non-admin update should overwrite app on_behalf_of_email with their own" + resp.status(), + 201, + "Should create app: {}", + resp.text().await? + ); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/users/rename/superadmin-external@windmill.dev" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "new_username": "superadmin_renamed" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "Should rename: {}", resp.text().await?); + + let app = sqlx::query!( + "SELECT policy FROM app WHERE path = $1 AND workspace_id = $2", + path, + "test-workspace" + ) + .fetch_one(&db) + .await?; + assert_eq!( + app.policy.get("on_behalf_of").and_then(|v| v.as_str()), + Some("u/superadmin_renamed"), + "the rename should follow the principal an app names" ); Ok(()) @@ -2855,10 +2923,8 @@ async fn test_reject_reserved_sentinel_on_behalf_of(db: Pool) -> anyho resp.text().await? ); - // App: a real superadmin on_behalf_of is *allowed* at deploy (deployers may - // deploy on behalf of any real user). The escalation is closed at execution - // by the job-token cap, not by restricting what can be stored, so even a - // superadmin email pinned onto an unrelated principal deploys fine here. + // App: a real superadmin's address pinned onto an unrelated principal is a pair naming two + // accounts, which the principal-authoritative policy refuses. let resp = authed( client().post(format!("{base}/apps/create")), "DEPLOYER_TOKEN", @@ -2873,12 +2939,14 @@ async fn test_reject_reserved_sentinel_on_behalf_of(db: Pool) -> anyho .await?; assert_eq!( resp.status(), - 201, - "a real superadmin on_behalf_of is allowed at deploy (capped at execution): {}", + 400, + "a superadmin address beside an unrelated principal must be refused: {}", resp.text().await? ); - // App: a consistently named real superadmin identity is likewise allowed. + // App: a real superadmin on_behalf_of is allowed at deploy when consistently named (deployers + // may deploy on behalf of any real user); the escalation is closed at execution by the + // job-token cap, not by restricting what can be stored. let resp = authed( client().post(format!("{base}/apps/create")), "DEPLOYER_TOKEN", diff --git a/backend/tests/relock_noop.rs b/backend/tests/relock_noop.rs new file mode 100644 index 0000000000..a480fc57ae --- /dev/null +++ b/backend/tests/relock_noop.rs @@ -0,0 +1,439 @@ +// These pin language-independent behaviour, and Python is the cheapest runtime that still +// generates a real lock out of relative imports, so the whole file needs that feature. +#![cfg(feature = "python")] + +use sqlx::{Pool, Postgres}; +use tokio_stream::StreamExt; +use windmill_api_client::types::NewScript; +use windmill_common::scripts::{deploy_relocked_version, fetch_script_for_update}; +use windmill_test_utils::*; + +const W: &str = "test-workspace"; + +/// Budget for one wait below, counted completions and drain together. Even against a cold cache +/// these settle in a few seconds, so it only bounds a step that is stuck, and it stays under the +/// 60s cap `in_test_worker` puts on the whole body so the panic names what was being waited on +/// rather than surfacing as a worker timeout. +const WAIT_BUDGET: std::time::Duration = std::time::Duration::from_secs(30); + +/// How often the waits below re-check. The drain reads the queue once per completion it waits +/// this long for, so it doubles as the floor on one turn of that loop. +const POLL: std::time::Duration = std::time::Duration::from_millis(20); + +const A: &str = "def main():\n return 'a'\n"; +const A_COMMENTED: &str = "# same dependencies, different content\ndef main():\n return 'a'\n"; +const A_WITH_TINY: &str = "import tiny\n\ndef main():\n return 'a'\n"; +const B: &str = "from f.rel.a import main as a\n\ndef main():\n return 'b' + a()\n"; +const C: &str = "from f.rel.b import main as b\n\ndef main():\n return 'c' + b()\n"; + +fn py_script(path: &str, content: &str, parent_hash: Option) -> NewScript { + NewScript { + draft_only: None, + content: content.into(), + language: windmill_api_client::types::ScriptLang::Python3, + lock: None, + parent_hash, + path: path.into(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + envs: vec![], + is_template: None, + kind: None, + summary: "".to_string(), + tag: None, + schema: std::collections::HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_secs: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + auto_kind: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + assets: vec![], + modules: None, + } +} + +#[derive(sqlx::FromRow, Debug)] +struct Version { + hash: i64, + archived: bool, + lock: Option, + created_at: chrono::DateTime, +} + +/// Every version of `path`, oldest first. +async fn versions(db: &Pool, path: &str) -> Vec { + sqlx::query_as( + "SELECT hash, archived, lock, created_at FROM script + WHERE workspace_id = $1 AND path = $2 ORDER BY created_at", + ) + .bind(W) + .bind(path) + .fetch_all(db) + .await + .unwrap() +} + +fn live(versions: &[Version]) -> &Version { + versions.iter().rev().find(|v| !v.archived).unwrap() +} + +/// `(path, status, logs)` of every dependency job created after `since`, in completion order. +async fn dependency_jobs_since( + db: &Pool, + since: chrono::DateTime, +) -> Vec<(String, String, String)> { + sqlx::query_as( + "SELECT j.runnable_path, c.status::text, COALESCE(l.logs, '') FROM v2_job_completed c + JOIN v2_job j ON j.id = c.id + LEFT JOIN job_logs l ON l.job_id = c.id + WHERE j.kind = 'dependencies' AND j.created_at > $1 + ORDER BY c.started_at", + ) + .bind(since) + .fetch_all(db) + .await + .unwrap() +} + +async fn wait_for_jobs( + db: &Pool, + completed: &mut (impl futures::Stream + Unpin), + count: usize, +) { + let deadline = tokio::time::Instant::now() + WAIT_BUDGET; + for i in 0..count { + tokio::time::timeout_at(deadline, completed.next()) + .await + .unwrap_or_else(|_| panic!("only {i} of {count} jobs completed")); + } + // Then let anything else that was queued run out, so a job the assertions say must not + // exist would have shown up here. A dependency job queues its fan-out before it completes, + // so an empty queue is a fixpoint rather than a lull. + loop { + while let Ok(Some(_)) = tokio::time::timeout(POLL, completed.next()).await {} + let queued: i64 = sqlx::query_scalar("SELECT count(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap(); + if queued == 0 { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "the queue never emptied" + ); + } +} + +/// A redeploy of an imported script whose dependencies did not move relocks its importer, +/// and that relock must deploy nothing: no new version, and no dependency job for the +/// importer's own importers. A redeploy that does change the dependencies still walks the +/// whole chain with a new version at each step. +#[sqlx::test(fixtures("base"))] +async fn relative_import_relock_deploys_only_when_the_lock_changed( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + // One at a time: each deploy's dependency job records the importer's edges, and an + // importer whose edges are recorded is what a later relock of it can skip on. + for (path, content) in [("f/rel/a", A), ("f/rel/b", B), ("f/rel/c", C)] { + client + .create_script(W, &py_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + } + let b_before = versions(&db, "f/rel/b").await; + let c_before = versions(&db, "f/rel/c").await; + assert_eq!(b_before.len(), 1); + assert_eq!(c_before.len(), 1); + + // Content-only change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!( + paths, + ["f/rel/a", "f/rel/b"], + "the leaf's own job and one no-op relock of its importer, and nothing for c" + ); + assert!( + jobs[1] + .2 + .contains("Lock unchanged: no new version deployed"), + "b's relock should have found its lock unchanged: {}", + jobs[1].2 + ); + let b_after = versions(&db, "f/rel/b").await; + let c_after = versions(&db, "f/rel/c").await; + assert_eq!( + b_after.len(), + 1, + "an unchanged relock must not mint a version" + ); + assert_eq!(live(&b_after).hash, live(&b_before).hash); + assert_eq!(c_after.len(), 1); + assert_eq!(live(&c_after).hash, live(&c_before).hash); + + // A dependency change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_WITH_TINY, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/c"]); + for path in ["f/rel/b", "f/rel/c"] { + let vs = versions(&db, path).await; + assert_eq!( + vs.len(), + 2, + "{path}: a changed relock deploys a new version" + ); + assert!( + vs[0].archived && !vs[1].archived, + "{path}: parent archived, child live" + ); + assert!(vs[0].created_at < vs[1].created_at, "{path}: lineage order"); + assert!( + vs[1].lock.as_deref().unwrap_or("").contains("tiny"), + "{path}: the new version carries the new lock: {:?}", + vs[1].lock + ); + } + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock that has to wait on its head's row lock, because a deploy of the same path holds +/// it, must find the version that deploy left and requeue itself for it rather than fail. The +/// blocked statement re-checks only the row it selected, which the deploy archived, and comes +/// back empty; the successor is only visible to a fresh read. +#[sqlx::test(fixtures("base"))] +async fn relock_waiting_on_a_deploy_requeues_for_its_successor( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + for (path, content) in [("f/rel/a", A), ("f/rel/b", B)] { + client + .create_script(W, &py_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + } + + // A deploy of b that holds its head's row lock for as long as this transaction lives. + let mut deploy = db.begin().await.unwrap(); + let head = fetch_script_for_update("f/rel/b", W, &mut *deploy) + .await + .unwrap() + .unwrap(); + + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + + // b's relock skips generation and reaches its commit, where it waits on the lock. + let deadline = std::time::Instant::now() + WAIT_BUDGET; + let mut waiting = false; + while !waiting && std::time::Instant::now() < deadline { + waiting = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock' + AND query LIKE '%FROM script WHERE path = $1%FOR UPDATE%')", + ) + .fetch_one(&db) + .await + .unwrap(); + if !waiting { + tokio::time::sleep(POLL).await; + } + } + assert!(waiting, "b's relock never reached the row lock"); + + // The deploy lands: the head is archived and a successor with its own lock takes + // its place, while the relock is still waiting. + let lock = head.lock.clone().unwrap(); + let successor = + deploy_relocked_version(&mut deploy, head, None, Some(&lock), None, None) + .await + .unwrap(); + deploy.commit().await.unwrap(); + + // a's own job, the relock that waited, and the relock it queued for the successor. + wait_for_jobs(&db, &mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/b"], "{jobs:?}"); + assert!( + jobs.iter().all(|(_, status, _)| status == "success"), + "no relock may fail on the wait: {jobs:?}" + ); + assert!( + jobs[1] + .2 + .contains("was deployed while this lock was generated"), + "the waiting relock should have seen the successor: {}", + jobs[1].2 + ); + assert!( + jobs[2] + .2 + .contains("Lock unchanged: no new version deployed"), + "the requeued relock should find the successor's lock current: {}", + jobs[2].2 + ); + let vs = versions(&db, "f/rel/b").await; + assert_eq!( + vs.len(), + 2, + "the deploy's successor and nothing else: {vs:?}" + ); + assert_eq!(live(&vs).hash, successor); + }, + port, + ) + .await; + + Ok(()) +} + +/// A multi-file importer: on a skipped relock each module gets its own last lock back, not the +/// parent script's, so an import's content-only redeploy leaves the importer alone as well. +#[sqlx::test(fixtures("base"))] +async fn multi_file_importer_relock_is_a_no_op_too(db: Pool) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + let py = |path: &str, content: &str, parent_hash: Option, with_module: bool| { + let mut ns = py_script(path, content, parent_hash); + if with_module { + ns.modules = Some(std::collections::HashMap::from([( + "helper.py".to_string(), + serde_json::json!({ + "content": "def greet(x):\n return 'hi ' + x\n", + "language": "python3" + }), + )])); + } + ns + }; + async fn module_lock(db: &Pool) -> Option { + sqlx::query_scalar( + "SELECT modules->'helper.py'->>'lock' FROM script + WHERE workspace_id = $1 AND path = 'f/rel/pb' AND archived = false", + ) + .bind(W) + .fetch_one(db) + .await + .unwrap() + } + + in_test_worker( + &db, + async { + client + .create_script(W, &py("f/rel/pa", "def main():\n return 'a'\n", None, false)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + client + .create_script( + W, + &py( + "f/rel/pb", + "from f.rel.pa import main as a\nfrom .helper import greet\n\ndef main():\n return greet(a())\n", + None, + true, + ), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + let lock_before = module_lock(&db).await; + assert!(lock_before.is_some(), "the module got a lock of its own on deploy"); + + let since = chrono::Utc::now(); + let pa_hash = live(&versions(&db, "f/rel/pa").await).hash; + client + .create_script( + W, + &py( + "f/rel/pa", + "# same dependencies\ndef main():\n return 'a'\n", + Some(format!("{pa_hash:016x}")), + false, + ), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/pa", "f/rel/pb"], "{jobs:?}"); + assert!( + jobs[1].2.contains("Lock unchanged: no new version deployed"), + "the multi-file importer's relock should be a no-op: {}", + jobs[1].2 + ); + assert_eq!(versions(&db, "f/rel/pb").await.len(), 1); + assert_eq!(module_lock(&db).await, lock_before, "the module keeps its own lock"); + }, + port, + ) + .await; + + Ok(()) +} diff --git a/backend/tests/relock_skip.rs b/backend/tests/relock_skip.rs index 8262a38cf4..bf5cb24cbf 100644 --- a/backend/tests/relock_skip.rs +++ b/backend/tests/relock_skip.rs @@ -266,7 +266,10 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + // Empty content leaves every importer's lock as it was, so only the five direct + // importers of the default deps run a job: an unchanged script relock deploys no + // version and so queues nothing for its own importers. + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; // Note: within a cascade, the same script may be triggered multiple times. // After the first trigger relocks and stores the hash, subsequent triggers skip. @@ -295,7 +298,7 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; assert!( diff --git a/backend/tests/script_auto_parent_archived.rs b/backend/tests/script_auto_parent_archived.rs index 5115c516c2..58b926b9c4 100644 --- a/backend/tests/script_auto_parent_archived.rs +++ b/backend/tests/script_auto_parent_archived.rs @@ -9,8 +9,9 @@ //! `lineage must be linear: no 2 scripts can have the same parent` error //! whenever that archived hash already had a child from the prior rename. //! -//! The fix clears `parent_hash` to `None` in that case so the push starts a -//! fresh lineage instead of failing. +//! Resolution only ever adopts an archived version nothing else descends from, +//! so here it finds none and leaves `parent_hash` at `None`, starting a fresh +//! lineage instead of failing. use serde_json::json; use sqlx::{Pool, Postgres}; @@ -146,3 +147,221 @@ async fn test_auto_parent_starts_fresh_lineage_when_all_versions_archived( Ok(()) } + +/// A path whose only versions are archived and childless — what archiving a path +/// leaves behind, and what a sync push that applies a deletion before the matching +/// update sees. Resolving `auto_parent` to no parent there hashes the deploy exactly +/// as the path's first version was hashed, so an unchanged push is rejected as a +/// duplicate of it. +#[sqlx::test(fixtures("base"))] +async fn test_auto_parent_adopts_archived_head_instead_of_colliding( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_archived_head"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let first_hash: i64 = + sqlx::query_scalar("SELECT hash FROM script WHERE path = $1 AND workspace_id = $2") + .bind(path) + .bind("test-workspace") + .fetch_one(&db) + .await?; + + let resp = authed( + client().post(format!("{base}/scripts/archive/p/{path}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "archiving the path should succeed"); + + // Byte-identical to the first deploy: the collision this guards against needs the + // pushed body to hash the same way the original one did. + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&{ + let mut push = body.clone(); + push["auto_parent"] = json!(true); + push + }) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "re-pushing an archived path must not collide with its own first version, \ + got {status}: {response_body}" + ); + + let active: Vec>> = sqlx::query_scalar( + "SELECT parent_hashes FROM script \ + WHERE path = $1 AND archived = false AND workspace_id = $2", + ) + .bind(path) + .bind("test-workspace") + .fetch_all(&db) + .await?; + assert_eq!(active.len(), 1, "exactly one active version expected"); + assert_eq!( + active[0].as_deref(), + Some(&[first_hash][..]), + "the revived version should continue the archived lineage" + ); + + Ok(()) +} + +/// The same redeploy from a caller that names no parent at all — the shape a retried +/// `wmill sync push` takes, once the archive it applied has committed and the path has +/// dropped out of the listing it diffs against. +/// +/// The adopted version supplies the lineage and nothing else: a path is reusable by a +/// different script, which must not start life holding grants nobody gave it. +#[sqlx::test(fixtures("base"))] +async fn test_parentless_redeploy_adopts_the_lineage_but_not_the_grants( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_retired_parentless"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let first_hash: i64 = + sqlx::query_scalar("SELECT hash FROM script WHERE path = $1 AND workspace_id = $2") + .bind(path) + .bind("test-workspace") + .fetch_one(&db) + .await?; + + sqlx::query("UPDATE script SET extra_perms = $1 WHERE hash = $2 AND workspace_id = $3") + .bind(json!({ "u/someone_else": true })) + .bind(first_hash) + .bind("test-workspace") + .execute(&db) + .await?; + + let resp = authed( + client().post(format!("{base}/scripts/archive/p/{path}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "archiving the path should succeed"); + + // Byte-identical to the first deploy and naming no parent: hashed as a first deploy it + // lands on the archived row. + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "a parentless redeploy of a retired path must not collide with its own first \ + version, got {status}: {response_body}" + ); + + let live: Vec<(Option>, serde_json::Value)> = sqlx::query_as( + "SELECT parent_hashes, extra_perms FROM script \ + WHERE path = $1 AND archived = false AND workspace_id = $2", + ) + .bind(path) + .bind("test-workspace") + .fetch_all(&db) + .await?; + assert_eq!(live.len(), 1, "exactly one live version expected"); + assert_eq!( + live[0].0.as_deref(), + Some(&[first_hash][..]), + "the redeploy should continue the retired lineage" + ); + assert_eq!( + live[0].1, + json!({}), + "an adopted version's grants must not carry over to whatever reuses its path" + ); + + Ok(()) +} + +/// A soft delete keeps the row, and with it the hash the version was deployed under, so a +/// tombstone has to stay adoptable: skip it and the redeploy hashes straight back onto it. +#[sqlx::test(fixtures("base"))] +async fn test_redeploy_chains_past_a_deleted_version(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_deleted_version"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let deleted_hash = resp.text().await?; + + let resp = authed( + client().post(format!("{base}/scripts/delete/h/{deleted_hash}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "deleting the version should succeed"); + + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "redeploying the content of a deleted version must not collide with its \ + tombstone, got {status}: {response_body}" + ); + + Ok(()) +} diff --git a/backend/tests/script_modules.rs b/backend/tests/script_modules.rs index 8755ebfb57..eec9d3b45a 100644 --- a/backend/tests/script_modules.rs +++ b/backend/tests/script_modules.rs @@ -159,3 +159,58 @@ export function main(name: string) { assert_eq!(result, json!("hello world")); Ok(()) } + +/// A multi-file script run without a lock is bundled by the lockfile build. A pinned import in +/// a workspace script it imports must be installed at that version and still resolve at run time. +#[sqlx::test(fixtures("base"))] +async fn test_bun_module_imports_pinned_workspace_script(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{}', '', '', 'f/system/pinned_module', 12350, 'bun', '')", + ) + .bind( + r#" +import _ from "lodash@4.17.20"; +export function lodashVersion() { return _.VERSION; } +"#, + ) + .execute(&db) + .await?; + + let mut modules = HashMap::new(); + modules.insert( + "helper.ts".to_string(), + ScriptModule { + content: "export function label(v: string) { return v; }\n".to_string(), + language: ScriptLang::Bun, + lock: None, + }, + ); + + let job = JobPayload::Code(RawCode { + content: r#" +import { lodashVersion } from "/f/system/pinned_module"; +import { label } from "./helper.ts"; +export function main() { return label(lodashVersion()); } +"# + .to_owned(), + path: Some("f/system/my_script".to_string()), + language: ScriptLang::Bun, + modules: Some(modules), + tag: None, + ..RawCode::default() + }); + + let result = RunJob::from(job) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!("4.17.20")); + Ok(()) +} diff --git a/backend/tests/trigger_listener_queries.rs b/backend/tests/trigger_listener_queries.rs index 1463138167..0af99136e7 100644 --- a/backend/tests/trigger_listener_queries.rs +++ b/backend/tests/trigger_listener_queries.rs @@ -178,6 +178,7 @@ fn make_authed() -> windmill_api_auth::ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs index 8ecf6e98b6..e93979e9c7 100644 --- a/backend/tests/volume_tests.rs +++ b/backend/tests/volume_tests.rs @@ -537,6 +537,64 @@ fn test_asset_kind_volume_variant() { #[cfg(feature = "parquet")] #[sqlx::test(fixtures("base"))] async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + let code = r#"// volume: test-vol data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("data/hello.txt", "utf-8"); + writeFileSync("data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("data/output.txt"), + }; +}"#; + run_volume_with_default_stack(db, ScriptLang::Bun, code).await +} + +#[cfg(all(feature = "parquet", feature = "private", feature = "php"))] +#[sqlx::test(fixtures("base"))] +async fn test_php_volume_with_default_stack(db: Pool) -> anyhow::Result<()> { + // Nested calls exercise the parser's fallback chain; flattening them weakens this guard. + let code = r#" $content, + "output_exists" => in_array("output.txt", array_values(array_diff(scandir("data"), ['.', '..']))), + ]; +}"#; + + run_volume_with_default_stack(db, ScriptLang::Php, code).await +} + +#[cfg(feature = "parquet")] +async fn run_volume_with_default_stack( + db: Pool, + language: ScriptLang, + code: &'static str, +) -> anyhow::Result<()> { + // CI raises RUST_MIN_STACK; keep the worker at Tokio's default to catch regressions. + tokio::task::spawn_blocking(move || { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_stack_size(2 * 1024 * 1024) + .enable_all() + .build()? + .block_on(run_volume_sql_worker_e2e(db, language, code)) + }) + .await? +} + +#[cfg(feature = "parquet")] +async fn run_volume_sql_worker_e2e( + db: Pool, + language: ScriptLang, + code: &str, +) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -571,24 +629,11 @@ async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; // 3. Push the job and run with SQL-connected worker - let code = r#"// volume: test-vol data - -import { readFileSync, writeFileSync, existsSync } from "fs"; - -export function main() { - const content = readFileSync("data/hello.txt", "utf-8"); - writeFileSync("data/output.txt", "written by sql worker"); - return { - read_content: content, - output_exists: existsSync("data/output.txt"), - }; -}"#; - let job = JobPayload::Code(RawCode { hash: None, content: code.to_string(), path: None, - language: ScriptLang::Bun, + language, lock: None, cache_ttl: None, cache_ignore_s3_path: None, diff --git a/backend/tests/wac_suspend_started_at.rs b/backend/tests/wac_suspend_started_at.rs new file mode 100644 index 0000000000..5a83730d24 --- /dev/null +++ b/backend/tests/wac_suspend_started_at.rs @@ -0,0 +1,111 @@ +//! Guards what `suspend_wac_parent` promises: the `started_at` invariant documented on +//! it, the segment length it hands back for metering, and that it stands down for a +//! cancel already on the row. + +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_worker::wac_executor::{suspend_wac_parent, WacPark}; + +#[sqlx::test] +async fn wac_suspend_clears_started_at(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, started_at) \ + VALUES ($1, 'test-workspace', now(), true, now() - interval '4 days')", + ) + .bind(job_id) + .execute(&db) + .await?; + + let mut tx = db.begin().await?; + let WacPark::Parked(segment_ms) = + suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await? + else { + panic!("an uncancelled parent must park"); + }; + tx.commit().await?; + + // The segment is what gets billed, so it must be the run that just ended, measured + // from the pull — not the park ahead of it, and not zero. + let four_days_ms = 4 * 24 * 3600 * 1000; + assert!( + segment_ms.is_some_and(|ms| (ms - four_days_ms).abs() < 60_000), + "expected the ended segment (~{four_days_ms}ms), got {segment_ms:?}" + ); + + let (started_at, running, suspend, suspend_until): ( + Option>, + bool, + i32, + Option>, + ) = sqlx::query_as( + "SELECT started_at, running, suspend, suspend_until FROM v2_job_queue WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + + assert_eq!( + started_at, None, + "a parked parent must not carry the previous segment's started_at" + ); + assert_eq!(suspend, 1); + assert!(suspend_until.is_some()); + assert!( + running, + "running stays true so the normal pull query skips the parked row" + ); + + Ok(()) +} + +/// A soft cancel sets `canceled_by` and `suspend = 0` and leaves acting on it to the next +/// pull. Parking over that holds the row until `suspend_until` — a whole day on a +/// `sleep(86400)` — so the park has to stand down and let the job complete instead. +#[sqlx::test] +async fn wac_suspend_stands_down_for_a_cancel(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job_queue \ + (id, workspace_id, scheduled_for, running, started_at, suspend, canceled_by, canceled_reason) \ + VALUES ($1, 'test-workspace', now(), true, now() - interval '30 seconds', 0, 'alice', 'no longer needed')", + ) + .bind(job_id) + .execute(&db) + .await?; + + let mut tx = db.begin().await?; + let parked = suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 86400.0).await?; + tx.commit().await?; + + match &parked { + WacPark::Cancelled(cancel) => { + assert_eq!(cancel.username.as_deref(), Some("alice")); + assert_eq!(cancel.reason.as_deref(), Some("no longer needed")); + } + other => panic!("a cancelled parent must not park, got {other:?}"), + } + + let (suspend, suspend_until, started_at): ( + i32, + Option>, + Option>, + ) = sqlx::query_as( + "SELECT suspend, suspend_until, started_at FROM v2_job_queue WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + + assert_eq!(suspend, 0, "the cancel's suspend = 0 must survive"); + assert_eq!( + suspend_until, None, + "a suspend_until would hold the row back for the whole park window" + ); + assert!( + started_at.is_some(), + "the segment ran, so its start must stay for the completion's duration" + ); + + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 2d3f1373fe..23d88301b7 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -318,19 +318,35 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho resp.text().await? ); } - // ...and the one `settings/global` key on the allowlist, which the CLI reads before - // creating a user on a git-sync push. `ws_base_url` is the control: the handler leaves - // it as ungated as `automate_username_creation`, so only the allowlist stops it. + // ...and the `settings/global` keys on the allowlist, which the CLI reads from a job: on a + // git-sync push, and in `u/admin/hub_sync`. `ws_base_url` is the control: the handler + // leaves it as ungated as these, so only the allowlist stops it. + for key in ["automate_username_creation", "uid", "hub_base_url"] { + let resp = authed( + client().get(format!("{api}/settings/global/{key}")), + &user_wm, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "WM_TOKEN must still read {key}: {}", + resp.text().await? + ); + } + // The same hub pull reads `hub_api_secret` for a private Hub, but a secret stays out of + // a job's reach even when the token borrows a superadmin. let resp = authed( - client().get(format!("{api}/settings/global/automate_username_creation")), - &user_wm, + client().get(format!("{api}/settings/global/hub_api_secret")), + &sa_wm, ) .send() .await?; - assert_eq!( - resp.status(), - 200, - "WM_TOKEN must still read automate_username_creation: {}", + let status = resp.status().as_u16(); + assert!( + status == 401 || status == 403, + "superadmin WM_TOKEN must not read hub_api_secret ({status}): {}", resp.text().await? ); let resp = authed( @@ -1094,6 +1110,7 @@ async fn test_privilege_gates_reject_a_job_token_directly( token_prefix: None, read_only: false, job_id, + credential_expiry: None, } } diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 419be6fdf3..3e2bae5092 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3091,16 +3091,16 @@ async fn test_php_job(db: Pool) -> anyhow::Result<()> { let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); - let content = r#" + let content = r#"// schema_validation ) -> anyhow: let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) - .email("test2@windmill.dev") + .as_user("test-user-2", "test2@windmill.dev") .run_until_complete(&db, false, server.addr.port()) .await; diff --git a/backend/tests/ws_specific.rs b/backend/tests/ws_specific.rs index cc15161381..322de598b7 100644 --- a/backend/tests/ws_specific.rs +++ b/backend/tests/ws_specific.rs @@ -391,7 +391,8 @@ async fn test_create_resource_upsert_clears_ws_specific(db: Pool) -> a /// Regression for GHSA-xmr2-98m6-cjf7: a token scoped only to `resources:write:` /// must NOT use the resource-delete cascade to delete a linked secret variable it has -/// no `variables:write` scope for. +/// no `variables:write` scope for. The victim sits at a path the resource owns, which is +/// the only kind the cascade reaches at all. #[sqlx::test(fixtures("ws_specific"))] async fn test_scoped_token_cannot_cascade_delete_linked_variable( db: Pool, @@ -406,7 +407,7 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( "SECRET_TOKEN", ) .json(&json!({ - "path": "u/test-user/victim_secret", + "path": "u/test-user/db_victim_secret", "value": "hunter2", "is_secret": true, "description": "" @@ -421,7 +422,7 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( ) .json(&json!({ "path": "u/test-user/db", - "value": { "password": "$var:u/test-user/victim_secret" }, + "value": { "password": "$var:u/test-user/db_victim_secret" }, "resource_type": "object" })) .send() @@ -443,9 +444,206 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( resp.text().await? ); assert!( - variable_exists(&db, "test-workspace", "u/test-user/victim_secret").await?, + variable_exists(&db, "test-workspace", "u/test-user/db_victim_secret").await?, "victim variable must survive the denied cascade" ); + // The scope check runs after the resource DELETE, so only the rollback keeps the resource + // alive — moving the check out of the transaction would silently delete it on a 403. + let resource_left: Option = + sqlx::query_scalar("SELECT COUNT(*) FROM resource WHERE workspace_id = $1 AND path = $2") + .bind("test-workspace") + .bind("u/test-user/db") + .fetch_one(&db) + .await?; + assert_eq!( + resource_left.unwrap_or(0), + 1, + "the denied delete must roll the resource back too" + ); + + Ok(()) +} + +/// Deleting a resource must not take a variable other things still need. Two gates, each +/// with a way past the other: a variable outside the resource's own path is never its to +/// delete, and even one it owns stays if another resource points at it. +#[sqlx::test(fixtures("ws_specific"))] +async fn test_resource_delete_spares_variables_it_does_not_own( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let create_var = |path: &'static str| { + authed( + client().post(format!("{base}/variables/create")), + "SECRET_TOKEN", + ) + .json(&json!({ "path": path, "value": "hunter2", "is_secret": true, "description": "" })) + .send() + }; + let create_res = |path: &'static str, var: &'static str| { + authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": path, + "value": { "password": format!("$var:{var}") }, + "resource_type": "object" + })) + .send() + }; + + // A shared secret at a path of its own, and two resources reading it. + assert_eq!(create_var("u/test-user/shared_canary").await?.status(), 201); + assert_eq!( + create_res("u/test-user/probe_a", "u/test-user/shared_canary") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/probe_b", "u/test-user/shared_canary") + .await? + .status(), + 201 + ); + + // A secret the resource at the same path owns, which a second resource also reads. + assert_eq!(create_var("u/test-user/owned").await?.status(), 201); + assert_eq!( + create_res("u/test-user/owned", "u/test-user/owned") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/borrower", "u/test-user/owned") + .await? + .status(), + 201 + ); + + for resource in ["u/test-user/probe_a", "u/test-user/owned"] { + let resp = authed( + client().delete(format!("{base}/resources/delete/{resource}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "delete {resource}: {}", + resp.text().await? + ); + } + + assert!( + variable_exists(&db, "test-workspace", "u/test-user/shared_canary").await?, + "a variable the deleted resource only referenced must survive" + ); + assert!( + variable_exists(&db, "test-workspace", "u/test-user/owned").await?, + "an owned variable another resource still references must survive" + ); + + Ok(()) +} + +/// The bulk cascade follows what RLS actually deleted, not what the caller asked for: a +/// resource the request names but leaves standing neither cascades nor stops counting as a +/// referrer. Both halves matter, and neither covers the other. +#[sqlx::test(fixtures("ws_specific"))] +async fn test_bulk_delete_follows_what_rls_deleted(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let create_var = |path: &'static str| { + authed( + client().post(format!("{base}/variables/create")), + "SECRET_TOKEN", + ) + .json(&json!({ "path": path, "value": "hunter2", "is_secret": true, "description": "" })) + .send() + }; + // ws_specific so the flag assertion at the end has something to check. + let create_res = |path: &'static str, var: &'static str| { + authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": path, + "value": { "password": format!("$var:{var}") }, + "resource_type": "object", + "ws_specific": true + })) + .send() + }; + + // Private to test-user: a resource and the secret it owns. + assert_eq!(create_var("u/test-user/hidden_pwd").await?.status(), 201); + assert_eq!( + create_res("u/test-user/hidden", "u/test-user/hidden_pwd") + .await? + .status(), + 201 + ); + // test-user-2's own resource and secret, which the private resource above also reads. + assert_eq!(create_var("u/test-user-2/own_pwd").await?.status(), 201); + assert_eq!( + create_res("u/test-user-2/own", "u/test-user-2/own_pwd") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/reader", "u/test-user-2/own_pwd") + .await? + .status(), + 201 + ); + + // test-user-2 may write the private secret but has no access to its resource at all. + sqlx::query( + "UPDATE variable SET extra_perms = '{\"u/test-user-2\": true}'::jsonb + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/hidden_pwd'", + ) + .execute(&db) + .await?; + + let resp = authed( + client().delete(format!("{base}/resources/delete_bulk")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "paths": ["u/test-user/hidden", "u/test-user-2/own", "u/test-user/reader"] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "bulk delete: {}", resp.text().await?); + + assert!( + variable_exists(&db, "test-workspace", "u/test-user/hidden_pwd").await?, + "the variable of a resource RLS refused to delete must survive" + ); + assert!( + variable_exists(&db, "test-workspace", "u/test-user-2/own_pwd").await?, + "a requested resource RLS left standing still counts as a referrer" + ); + // ws_specific has no RLS policy of its own, so clearing it by requested path rather than + // by deleted path would quietly turn a surviving resource workspace-generic. + assert_eq!( + ws_specific_row_count(&db, "test-workspace", "resource", "u/test-user/hidden").await?, + 1, + "a resource RLS refused to delete must keep its ws_specific flag" + ); Ok(()) } diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 09720b2203..89e3c7c6b2 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -6,6 +6,7 @@ use axum::{ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::Row; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use windmill_common::{ assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind}, db::UserDB, @@ -13,7 +14,9 @@ use windmill_common::{ utils::escape_ilike_pattern, }; -use windmill_api_auth::{build_scope_path_predicate, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_filter, build_scope_path_predicate, ApiAuthed, ScopePathFilter, +}; // Partition-range backfill preview. The logic (producer resolution, range // enumeration, status join) is enterprise: the `private` build compiles the @@ -33,6 +36,7 @@ pub fn workspaced_service() -> Router { .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) .route("/graph", get(asset_graph)) + .route("/column_lineage", get(dbt_column_lineage)) .route("/pipelines", get(list_pipeline_folders)) .route("/partitions", get(list_partitions)) .route("/partitions_in_range", get(list_partitions_in_range)) @@ -663,10 +667,21 @@ struct DbtAssetProvenance { description: Option, #[serde(skip_serializing_if = "Vec::is_empty")] data_tests: Vec, - /// Declared column metadata (name -> description). NOT column lineage — - /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14). + /// Declared column metadata (name -> description): what `manifest.json` + /// carries, which is only the columns an author wrote down. #[serde(skip_serializing_if = "Option::is_none")] columns: Option, + /// Every column of the relation, typed and in order — + /// `[{"name": …, "type": …}]` — from the engine's static analysis. Present + /// only for a project that opted into it. + /// + /// Gated exactly like `columns` and the model's SQL: a full column list is + /// the shape of what the author WROTE, one level finer than the `ref()` + /// graph, which is ungated only because it draws relations the caller + /// already sees in `asset`. Widening that boundary has to be a decision, not + /// a consequence of a project turning the analysis pass on. + #[serde(skip_serializing_if = "Option::is_none")] + column_schema: Option, /// A source's declared freshness policy, for the staleness chip. #[serde(skip_serializing_if = "Option::is_none")] freshness: Option, @@ -946,6 +961,458 @@ struct DbtLineageEdge { to_asset_path: String, } +/// One column-to-column edge, in the same terms: the two relations and the two +/// columns, never dbt's node ids. +#[derive(Serialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct DbtColumnLineageEdge { + from_asset_path: String, + from_column: String, + to_asset_path: String, + to_column: String, + /// dbt's own word for how the value travelled: `copy` (passthrough), `mod` + /// (transformed), `scan` (read to produce the ROW rather than the value — a + /// join key, a predicate, a `group by`). Sent verbatim, including a kind + /// this engine version invented, because the renderer decides what a kind + /// means and the set is the engine's. + kind: String, +} + +/// The dbt relations a view is tracing, and which stored graph to read them +/// from. +/// +/// Several relations, answered as one union, because ONE selection reaches +/// several: a script's output column can be derived from columns of several dbt +/// models, and a model's columns can be consumed by scripts that feed others. +/// Asking per relation instead is a request per boundary plus the bookkeeping to +/// stitch the answers together and decide which of them is still current — which +/// is a cache, and is what taking them together exists to not need. +pub struct ColumnLineageQuery { + /// The `dbt://` relations whose lineage to return. + pub asset_paths: Vec, + /// The deployed version a view is drawing, when it is drawing one — the dbt + /// editor, which shows a single project as of a single deploy. + /// + /// A version-pinned answer is that version's project ALONE, the same as a + /// job-pinned one: the pin exists so the trace describes the stored graph on + /// screen, and another project's live graph is not part of it. Only the + /// unpinned answer crosses projects. + /// + /// A run's or an editor buffer's graph is NOT reachable from here: it pins + /// to a job, and that costs the job-read gate. + pub dbt_script_hash: Option, +} + +impl ColumnLineageQuery { + /// Built from the raw pairs rather than deserialized as a struct, because + /// `asset_path` REPEATS and `serde_urlencoded` — what `Query` deserializes + /// with — reads no sequence from a repeated key. A GET rather than a POST + /// body carrying the list: the method decides a scoped token's action, so a + /// POST would ask `assets:write` for a read and refuse a read-only token + /// outright. + pub fn from_query_pairs(pairs: Vec<(String, String)>) -> windmill_common::error::Result { + let mut asset_paths: Vec = Vec::new(); + let mut dbt_script_hash = None; + for (key, value) in pairs { + match key.as_str() { + "asset_path" => asset_paths.push(value), + // Hex, like every other script-hash parameter, so a page can + // pass `job.script_hash` verbatim. + "dbt_script_hash" => { + dbt_script_hash = + Some(serde_json::from_value(Value::String(value)).map_err(|_| { + windmill_common::error::Error::BadRequest( + "dbt_script_hash is not a script hash".to_string(), + ) + })?) + } + _ => {} + } + } + // REFUSED, not answered empty. A caller that named no relation — or + // misspelled the parameter — asked for something, and an empty component + // is what a relation with no lineage returns, so answering that way says + // "this has none" for a question that was never asked. + if asset_paths.is_empty() { + return Err(windmill_common::error::Error::BadRequest( + "at least one asset_path is required".to_string(), + )); + } + if asset_paths.len() > MAX_ASKED_RELATIONS { + return Err(windmill_common::error::Error::BadRequest(format!( + "at most {MAX_ASKED_RELATIONS} asset_path values may be asked about at once" + ))); + } + Ok(ColumnLineageQuery { asset_paths, dbt_script_hash }) + } +} + +#[derive(Serialize, Debug, PartialEq, Eq)] +pub struct ColumnLineageResponse { + /// Direct (`copy` / `mod`) column edges of the component the asked-for + /// relations' columns sit in, in the terms the canvas draws. Empty when no + /// project involved asked for the analysis pass, which is the ordinary case. + edges: Vec, + /// The component reaches further than what is here: `edges` holds the part + /// nearest the asked-for relations. Said rather than silently cut, because a + /// trace that stops short is otherwise indistinguishable from one that ends. + truncated: bool, +} + +/// How many edges one trace may carry back. The renderer draws a box per column, +/// so a component past this is unreadable however it is served — a synthetic +/// 3000-model project whose models share a column returns 58k direct edges and +/// 7.3MB. Applied over the walk, which is the LAST filter, so what survives is +/// the part nearest the selection rather than an arbitrary slice of it. +const MAX_TRACE_EDGES: usize = 5_000; + +/// How many times one trace may discover a project it has not read yet. Each +/// round costs a gate and a fetch, and a component crossing this many projects +/// has already outgrown what the canvas can show; stopping says what the edge +/// bound says. +const MAX_OWNER_ROUNDS: usize = 8; + +/// How many edges a trace may already hold before it stops looking for projects +/// it has not read. NOT a bound on what one trace fetches: the seeds' own +/// projects are read whole whatever their size, because reading them IS the +/// answer, and a project's edges arrive whole in any case — the walk is what +/// decides which of them are in the component, so a `LIMIT` would cut a set that +/// need not contain the asked-for relation at all. What a single fetch is +/// bounded by is the ingest's `MAX_COLUMN_EDGES` per version. This bounds the +/// EXPANSION on top of that: reading a further project's worth once a trace is +/// already this size buys nothing the walk will not cut at `MAX_TRACE_EDGES`. +const EXPANSION_EDGE_BUDGET: usize = 100_000; + +/// How many relations one request may ask about. Generous: the pipeline page +/// sends every dbt relation the selection's own producer lineage reaches, which +/// is a handful even in a large folder. It exists so a crafted request cannot +/// hand `= ANY($2)` an arbitrarily long array. +const MAX_ASKED_RELATIONS: usize = 1_000; + +/// A stored project graph: a deployed version, or one job's snapshot of it. +/// `script_hash` is NULL for an editor buffer's parse, which names no version. +type ProjectVersion = (String, Option, uuid::Uuid); + +async fn dbt_column_lineage( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(pairs): Query>, +) -> JsonResult { + // `None`: pinning to one run is job-scoped and this endpoint is authorized + // as `assets:read`. See `dbt_column_lineage_for`. + let q = ColumnLineageQuery::from_query_pairs(pairs)?; + dbt_column_lineage_for(&authed, &w_id, user_db, q, None).await +} + +/// The column-level lineage the asked-for relations sit in, optionally as one +/// run saw it. +/// +/// AUTHORIZES NOTHING BY ITSELF, on the same contract as `asset_graph_for`: +/// `assets:read` always, and the job-read gate for `Some(pinned)`, whose path +/// and hash are then taken from that job's row rather than from the caller. +/// +/// A column trace is transitive and a relation is not owned by one project, so +/// the answer grows a project at a time: resolve who owns the relations reached +/// so far, read their edges, walk, and repeat for the relations that walk newly +/// reached. Every round re-applies the caller's gate to the projects it +/// discovers — a relation being reachable from a project the caller may read +/// says nothing about the project on the other side of it. +pub async fn dbt_column_lineage_for( + authed: &ApiAuthed, + w_id: &str, + user_db: UserDB, + q: ColumnLineageQuery, + pinned: Option, +) -> JsonResult { + // A column-level view is the shape of what the author WROTE, so it takes the + // model's own gate rather than the relation's. + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(authed, "scripts", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + let (pinned_path, script_hash) = match pinned.as_ref() { + // The job's own version, so a pin cannot name one project's run while + // claiming another's version — including when it names NONE, which is + // the editor buffer. + Some(p) => (Some(p.script_path.as_str()), p.script_hash), + None => (None, q.dbt_script_hash.map(|h| h.0)), + }; + let pinned_job_id = pinned.as_ref().map(|p| p.job_id); + + let seeds: BTreeSet = q.asset_paths.into_iter().collect(); + let mut tx = user_db.begin(authed).await?; + + // Relations whose owners have been asked for, project graphs already read, + // and the edges they yielded. These are what end the loop: a round asks only + // about relations not asked about before and reads only projects not read + // before, so it stops as soon as one of the two runs out. + let mut asked: HashSet = HashSet::new(); + let mut read: HashSet = HashSet::new(); + let mut edges: Vec = Vec::new(); + let mut answer: Vec = Vec::new(); + let mut pending: Vec = seeds.iter().cloned().collect(); + let mut truncated = false; + let mut rounds = 0usize; + + loop { + if pending.is_empty() { + break; + } + // Which project version owns each of these relations, under this + // caller's access. Usually one row per relation; a relation a second + // project declares as a source has two, and each answers for its own + // lineage. + let owners = sqlx::query!( + r#"SELECT DISTINCT n.script_path AS "script_path!", n.script_hash, n.job_id AS "job_id!" + FROM dbt_node n + WHERE n.workspace_id = $1 AND n.asset_path = ANY($2) + -- The run's snapshot, or the deployed graph when that job stored + -- none -- a build pins only if it wrote one. + AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $5) + THEN $5::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END + -- The gate, re-decided for every project the walk reaches. That + -- is what resolving owners in a loop is for: being entitled to + -- one project is not being entitled to the one that declares a + -- relation it hands over. + AND ( $6 + OR n.script_path = ANY($7) + OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx + WHERE n.script_path = pfx + OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) ) + AND CASE + -- Pinned: which version comes from a job this caller was + -- already granted, so `script` does not decide THAT -- but + -- it still decides whether the project may be read, the + -- same second gate `script_visible` is on the graph. Being + -- entitled to a run is not being entitled to the SQL + -- behind it, and column lineage is that SQL's shape. A + -- version-less row is exempt because it is an editor + -- buffer, which has no `script` row to ask and reaches + -- this only through the parse job that wrote it. + -- + -- One project answers, so a pinned trace never crosses + -- into another: neither does the graph it annotates. + WHEN $4::text IS NOT NULL + THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint + AND ($3::bigint IS NULL OR EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3)) + -- A named version: the deployed one an editor is drawing. + -- `script` is read under RLS, so this is the visibility + -- check as well as the existence one. A hash names one + -- script row, so this arm answers for one project too — + -- and deliberately: a pin says which stored graph is on + -- screen, and another project's live graph is not it. + WHEN $3::bigint IS NOT NULL + THEN n.script_hash = $3 AND EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3) + -- Otherwise the version deployed now: an older one's rows + -- outlive it in `dbt_node` until the sweep, and describe a + -- project that is no longer what runs. `language` narrows + -- it the way the graph's own resolution does, so a path + -- that has since become a script of another kind draws and + -- explains the same version rather than disagreeing. Read + -- under RLS, so a project the caller cannot see resolves + -- to NULL and matches nothing. + ELSE n.script_hash = ( + SELECT sc.hash FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.language = 'dbt' + AND sc.deleted = false AND sc.archived = false + ORDER BY sc.created_at DESC LIMIT 1) + END"#, + w_id, + &pending[..], + script_hash, + pinned_path, + pinned_job_id, + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + asked.extend(pending.drain(..)); + + let fresh: Vec = owners + .into_iter() + .map(|o| (o.script_path, o.script_hash, o.job_id)) + .filter(|k| read.insert(k.clone())) + .collect(); + // Nothing left this caller may read and has not read: the component is + // whole, however many relations were still waiting to be asked about. + // Their owners are projects already in hand. + if fresh.is_empty() { + break; + } + // From here a project exists that this answer will not contain, so the + // two stops below are cuts and are reported as such. Deciding it after + // the owners query rather than before is what keeps a big project's + // small component from being called truncated: `pending` alone only says + // a relation has not been ASKED about, not that anything was left out. + rounds += 1; + if rounds > MAX_OWNER_ROUNDS || edges.len() >= EXPANSION_EDGE_BUDGET { + truncated = true; + break; + } + let fresh_paths: Vec = fresh.iter().map(|k| k.0.clone()).collect(); + let fresh_hashes: Vec> = fresh.iter().map(|k| k.1).collect(); + let fresh_jobs: Vec = fresh.iter().map(|k| k.2).collect(); + + // DIRECT kinds only. `scan` -- the column was read to produce the ROW, + // not the value -- reaches every output column of its model, so it is + // most of a project's stored lineage and none of what a trace draws. + // It stays in the table for a later view to ask for. + let rows = sqlx::query!( + r#"SELECT p.asset_path AS "from_path!", e.parent_column AS "from_column!", + c.asset_path AS "to_path!", e.child_column AS "to_column!", + e.lineage_kind AS "kind!" + FROM unnest($2::text[], $3::bigint[], $4::uuid[]) + AS o(script_path, script_hash, job_id) + JOIN dbt_column_edge e ON e.workspace_id = $1 + AND e.script_path = o.script_path + AND e.job_id = o.job_id + -- `=` still, with the NULL-to-NULL case + -- spelled out and gated on the pin: a + -- version-less row's hash is NULL on both + -- sides, which `=` never matches, but + -- `IS NOT DISTINCT FROM` would cost the + -- equality its index bound everywhere else. + AND (e.script_hash = o.script_hash + OR ($5::text IS NOT NULL + AND o.script_hash IS NULL + AND e.script_hash IS NULL)) + JOIN dbt_node p ON p.workspace_id = e.workspace_id + AND p.script_path = e.script_path + AND p.script_hash IS NOT DISTINCT FROM e.script_hash + AND p.job_id = e.job_id + AND p.unique_id = e.parent_unique_id + JOIN dbt_node c ON c.workspace_id = e.workspace_id + AND c.script_path = e.script_path + AND c.script_hash IS NOT DISTINCT FROM e.script_hash + AND c.job_id = e.job_id + AND c.unique_id = e.child_unique_id + WHERE e.lineage_kind IN ('copy', 'mod') + AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL"#, + w_id, + &fresh_paths[..], + &fresh_hashes[..] as &[Option], + &fresh_jobs[..], + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + edges.extend(rows.into_iter().map(|r| DbtColumnLineageEdge { + from_asset_path: r.from_path, + from_column: r.from_column, + to_asset_path: r.to_path, + to_column: r.to_column, + kind: r.kind, + })); + // Two projects can describe one relation, so the same edge can arrive + // twice. Sorted as well as deduplicated: the walk reads the incidence + // lists in this order, so the answer does not depend on which round a + // project was discovered in. + edges.sort(); + edges.dedup(); + + let walked = component(&edges, &seeds); + answer = walked.edges; + truncated = walked.truncated; + if truncated { + break; + } + // The relations the walk newly reached. Their owners are the next round's + // question: this project declares them, and so may another. + pending = answer + .iter() + .flat_map(|e| [&e.from_asset_path, &e.to_asset_path]) + .filter(|p| !asked.contains(*p)) + .cloned() + .collect::>() + .into_iter() + .collect(); + } + tx.commit().await?; + Ok(Json(ColumnLineageResponse { truncated, edges: answer })) +} + +struct WalkedComponent { + edges: Vec, + truncated: bool, +} + +/// The edges of the connected component the asked-for relations sit in, nearest +/// first and at most `MAX_TRACE_EDGES` of them. +/// +/// The canvas lays out the component of the selected relation's columns, so a +/// project's other model families are edges nothing it draws can reach. Walked +/// here rather than in SQL: a recursive CTE has no index to walk, so it rescans +/// the whole edge set once per level — measured at 1.24s against 59ms for the +/// query alone on a 3000-model project, for a walk that is microseconds over a +/// map. Columns are keyed by relation, not by project, which is how the canvas +/// keys them too: two projects describing one relation draw one node. +/// +/// Breadth-first, so the bound cuts the far end of the trace rather than an +/// arbitrary part of it. +fn component(edges: &[DbtColumnLineageEdge], seeds: &BTreeSet) -> WalkedComponent { + let mut incident: HashMap<(&str, &str), Vec> = HashMap::new(); + for (i, e) in edges.iter().enumerate() { + incident + .entry((&e.from_asset_path, &e.from_column)) + .or_default() + .push(i); + incident + .entry((&e.to_asset_path, &e.to_column)) + .or_default() + .push(i); + } + let mut start: Vec<(&str, &str)> = incident + .keys() + .filter(|(path, _)| seeds.contains(*path)) + .copied() + .collect(); + start.sort(); + let mut seen_node: HashSet<(&str, &str)> = start.iter().copied().collect(); + let mut queue: VecDeque<(&str, &str)> = start.into(); + let mut taken = vec![false; edges.len()]; + let mut kept: Vec = Vec::new(); + let mut truncated = false; + 'walk: while let Some(node) = queue.pop_front() { + for &i in incident.get(&node).map(Vec::as_slice).unwrap_or_default() { + if std::mem::replace(&mut taken[i], true) { + continue; + } + if kept.len() == MAX_TRACE_EDGES { + truncated = true; + break 'walk; + } + kept.push(i); + let e = &edges[i]; + let ends = [ + (e.from_asset_path.as_str(), e.from_column.as_str()), + (e.to_asset_path.as_str(), e.to_column.as_str()), + ]; + for end in ends { + if seen_node.insert(end) { + queue.push_back(end); + } + } + } + } + // Back into edge order, so a response does not carry the walk's shape. + kept.sort_unstable(); + WalkedComponent { edges: kept.into_iter().map(|i| edges[i].clone()).collect(), truncated } +} + async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, @@ -1323,7 +1790,7 @@ pub async fn asset_graph_for( n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path, n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description, n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node, - n.columns, n.freshness, + n.columns, n.column_schema, n.freshness, n.raw_code, n.original_file_path, -- Whether the caller may read the project this row describes. -- The query deliberately reaches outside the requested folder @@ -1379,6 +1846,10 @@ pub async fn asset_graph_for( // `ref()` lineage between two models, resolved to the relations they // produce. Joined to `dbt_node` on both key columns because a dbt // `unique_id` is only unique within its project. + // + // Column lineage is NOT here. It is stored per relation and per column, and + // this response is folder-wide and polled by a run page, so it carries only + // what the canvas draws for every node at once. let dbt_edge_rows = sqlx::query!( r#"WITH live AS ( SELECT * FROM ( @@ -1604,6 +2075,7 @@ pub async fn asset_graph_for( description: r.description.clone().filter(|_| source_allowed), data_tests: vec![], columns: r.columns.clone().filter(|_| source_allowed), + column_schema: r.column_schema.clone().filter(|_| source_allowed), freshness: r.freshness.clone().filter(|_| source_allowed), }; // One relation can carry rows from several projects — typically a model diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs index 7f4979a96f..ebeee49ecc 100644 --- a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -8,7 +8,9 @@ //! whole dbt half, and every later fix in this area re-touched one of the two. use sqlx::{Pool, Postgres}; -use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun}; +use windmill_api_assets::{ + asset_graph_for, dbt_column_lineage_for, ColumnLineageQuery, GraphQuery, PinnedRun, +}; use windmill_api_auth::ApiAuthed; use windmill_common::db::UserDB; @@ -33,6 +35,7 @@ fn outsider() -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -83,6 +86,34 @@ async fn seed(db: &Pool, job: uuid::Uuid) { .execute(db) .await .unwrap(); + // The relation it reads, and the column edge between them. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders', + 'u/a/wh/analytics/raw_orders', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', + 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); // A test node, for the arguments it carries: `accepted_values` spells out a // column's domain. sqlx::query!( @@ -364,7 +395,25 @@ async fn seed_editor_graph(db: &Pool, job: uuid::Uuid) { r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, raw_code, tags) VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft', - 'u/a/wh/analytics/draft', 'select 3', '{}')"#, + 'u/a/wh/analytics/draft', 'select 3', '{}'), + ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src', + 'u/a/wh/analytics/draft_src', 'select 4', '{}')"#, + WS, + PATH, + job + ) + .execute(db) + .await + .unwrap(); + // A version-less row's `script_hash` is NULL on both sides of every join and + // every visibility check, and `= NULL` is never true — so the column edges + // need the same NULL arm the node query has, or a buffer parse renders its + // columns and none of their lineage. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", WS, PATH, job @@ -450,3 +499,515 @@ async fn an_editor_graph_renders_only_through_its_own_job(db: Pool) { "nor of a run of the deployed version: {deployed_run}" ); } + +async fn column_lineage_q( + db: &Pool, + authed: &ApiAuthed, + pairs: Vec<(String, String)>, + pinned: Option, +) -> serde_json::Value { + let res = dbt_column_lineage_for( + authed, + WS, + UserDB::new(db.clone()), + ColumnLineageQuery::from_query_pairs(pairs).unwrap(), + pinned, + ) + .await + .unwrap(); + serde_json::to_value(&res.0).unwrap() +} + +async fn column_lineage( + db: &Pool, + authed: &ApiAuthed, + asset_paths: &[&str], + pinned: Option, +) -> serde_json::Value { + let pairs = asset_paths + .iter() + .map(|p| ("asset_path".to_string(), p.to_string())) + .collect(); + column_lineage_q(db, authed, pairs, pinned).await +} + +/// The buffer parse's own lineage, which is the case the versionless rows exist +/// for. Its `script_hash` is NULL on both sides of every join and every +/// visibility check, and `= NULL` is never true — so the versionless arm has to +/// be written for it, or a parse renders its columns and none of their lineage. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_editor_buffers_column_lineage_answers_through_its_job(db: Pool) { + let parse = uuid::Uuid::from_u128(9); + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_editor_graph(&db, parse).await; + let admin = ApiAuthed { is_admin: true, ..outsider() }; + + let pinned = PinnedRun { job_id: parse, script_path: PATH.to_string(), script_hash: None }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], Some(pinned)).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/draft_src", + "from_column": "raw", + "to_asset_path": "u/a/wh/analytics/draft", + "to_column": "clean", + "kind": "mod", + }]), + ); + // Unpinned, the same relation resolves through the deployed version, which + // never heard of the buffer's models. + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], None).await["edges"], + serde_json::json!([]), + "a buffer's lineage is reachable only through the job that parsed it" + ); +} + +/// Being entitled to a RUN is not being entitled to the SQL behind it, and +/// column lineage is that SQL's shape. The pinned graph draws the relations for +/// a share-link viewer and redacts what the author wrote; the lineage is the +/// second, and resolving the version from the job must not be mistaken for +/// deciding that too. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_pinned_run_does_not_hand_over_the_projects_column_lineage(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + + assert_eq!( + column_lineage( + &db, + &outsider(), + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([]), + "the run renders for them, its column-level shape does not" + ); + assert_eq!( + column_lineage( + &db, + &ApiAuthed { is_admin: true, ..outsider() }, + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "while a reader of the project gets it" + ); +} + +/// The deployed version of the same two models, plus a `scan` edge beside the +/// direct one: `seed`'s rows are a run's snapshot, and the unpinned answer is +/// the version's own graph. +async fn seed_deployed_orders(db: &Pool) { + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders', + 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders', + 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); +} + +/// A column-level view is the shape of what the author WROTE, so it takes the +/// script's own gate — the same one that keeps `raw_code` behind access to the +/// project. `scan` says the column was read to produce the ROW rather than the +/// value, so it reaches every output column of its model and is never served. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_takes_the_scripts_gate_and_only_the_direct_kinds(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the direct edge, and not the `scan` one beside it" + ); + assert_eq!( + column_lineage(&db, &outsider(), &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([]), + "and nothing at all for a caller who cannot read the project" + ); +} + +/// One project routinely holds model families that share no column, and the +/// canvas lays out the connected component of the selected relation's columns. +/// Answering with the project's other components sends edges nothing can draw. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_stops_at_the_selected_relations_component(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + // A second family in the same project version, reaching neither of the two + // relations `seed` wired together. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock', + 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily', + 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], Some(pinned())).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the orders family, and not the stock one beside it in the same project" + ); + assert_eq!( + column_lineage( + &db, + &admin, + &["u/a/wh/analytics/stock_daily"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + "and the other way round — reached from the child end, which is upstream" + ); +} + +/// A deployed dbt project in `folder`, declaring `parent`'s relation as a source +/// and deriving `child` from it. `orders → mart → secret_out` is three projects +/// chained through two shared relations. +async fn seed_neighbour_project( + db: &Pool, + folder: &str, + hash: i64, + parent: (&str, &str), + child: (&str, &str), +) { + let path = format!("f/{folder}/proj"); + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) + VALUES ($1, $2, $2, '{}', '{}')", + WS, + folder + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + hash, + path, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4, + 'source', $4, $5, '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6, + 'model', $6, $7, '{}')", + WS, + path, + hash, + parent.0, + parent.1, + child.0, + child.1, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + WS, + path, + hash, + parent.0, + child.0, + ) + .execute(db) + .await + .unwrap(); +} + +/// A trace crosses out of the project it started in, and the gate crosses with +/// it. +/// +/// A relation one project produces is another's source, so the component reaches +/// edges the first project's owner set never named — that is what resolving +/// owners to a fixpoint is for. The other half is that the caller's access has +/// to be re-decided for each project discovered on the way: reaching a relation +/// says nothing about who may read the project on the far side of it. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_crosses_projects_only_where_the_caller_may_read_them(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + seed_neighbour_project( + &db, + "mid", + 43, + ("orders", "u/a/wh/analytics/orders"), + ("mart", "u/a/wh/analytics/mart"), + ) + .await; + seed_neighbour_project( + &db, + "secret", + 44, + ("mart", "u/a/wh/analytics/mart"), + ("secret_out", "u/a/wh/analytics/secret_out"), + ) + .await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let reached = |body: &serde_json::Value| { + body["edges"] + .as_array() + .unwrap() + .iter() + .map(|e| e["to_asset_path"].as_str().unwrap().to_string()) + .collect::>() + }; + + let all = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&all), + [ + "u/a/wh/analytics/mart", + "u/a/wh/analytics/orders", + "u/a/wh/analytics/secret_out" + ] + .map(String::from) + .into(), + "two projects out from the one asked about, not one: {all}" + ); + + // Granted the first two folders and not the third. The edges of the project + // they may read are theirs; the one beyond it is not, even though the + // relation joining them is in the answer. + let partial = ApiAuthed { + folders: vec![ + ("private".to_string(), false, false), + ("mid".to_string(), false, false), + ], + ..outsider() + }; + let some = column_lineage(&db, &partial, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&some), + ["u/a/wh/analytics/mart", "u/a/wh/analytics/orders"] + .map(String::from) + .into(), + "the trace stops where the caller's access does: {some}" + ); + + // The other half of the same gate, and the half that is hand-written SQL + // rather than RLS: a token scoped to one folder reaches the projects in it + // and no others, whatever its grants say. + let scoped = ApiAuthed { + is_admin: true, + scopes: Some(vec!["scripts:read:f/private/*".to_string()]), + ..outsider() + }; + let scoped = column_lineage(&db, &scoped, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&scoped), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "and where its scope does: {scoped}" + ); + + // A version pin answers for that version's project alone — the dbt editor, + // which draws one project as of one deploy. Crossing into `mid` here would + // annotate that canvas with relations it does not draw. + let pinned_version = column_lineage_q( + &db, + &admin, + vec![ + ( + "asset_path".to_string(), + "u/a/wh/analytics/orders".to_string(), + ), + ("dbt_script_hash".to_string(), format!("{:016x}", HASH)), + ], + None, + ) + .await; + assert_eq!( + reached(&pinned_version), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "a version pin does not cross into the project beside it: {pinned_version}" + ); +} + +/// The bound on the answer, and that hitting it is said rather than silently +/// cut: a trace that stops reads exactly like one that ends. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_component_past_the_bound_is_cut_and_says_so(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // One direct edge per column pair, more of them than a trace may carry. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 6000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!(body["edges"].as_array().unwrap().len(), 5000); + assert_eq!(body["truncated"], serde_json::json!(true)); +} + +/// The other side of the same flag: a project big enough to stop the expansion +/// still answers a small component WHOLE, and must not claim it was cut. +/// +/// Reaching the size that stops expansion says nothing on its own — nor does a +/// relation whose owners have not been asked about, since those owners are +/// usually the project already in hand. Only a project this caller may read and +/// this answer does not contain is a cut. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_whole_component_in_a_big_project_is_not_called_cut(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // A second family, two relations and one edge, sharing no column with the + // first — the whole of what a selection on it should return. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock', + 'model', 'stock', 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily', + 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + // And enough unrelated families beside them to pass the expansion budget. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 100000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/stock_daily"], None).await; + assert_eq!( + body["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + ); + assert_eq!(body["truncated"], serde_json::json!(false)); +} diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index bef59aea6c..6e5aa6b1da 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -138,12 +138,28 @@ impl AuthCache { w_id: Option, token: &str, ) -> Option { - let mut opt_job_authed = self.get_opt_job_authed_inner(w_id, token).await?; + let mut opt_job_authed = self.get_opt_job_authed_inner(w_id.clone(), token).await?; // Single source of truth: mirror the resolved job_id onto the authed so // every consumer (require_super_admin, ...) sees that this identity came // from a job's WM_TOKEN, even on an AUTH_CACHE hit whose cached authed // predates this field. opt_job_authed.authed.job_id = opt_job_authed.job_id; + // The workspace's guest switch is enforced here, once, for every guest request + // — not per handler, where each guest-reachable route would have to remember + // it. Uncached, so turning guests off takes effect on the next request of every + // guest session and every token derived from one. + if crate::scopes::has_guest_sentinel(opt_job_authed.authed.scopes.as_deref()) { + let Some(w_id) = w_id else { return None }; + let email = &opt_job_authed.authed.email; + match windmill_common::workspaces::guest_session_stands(&self.db, &w_id, email).await { + Ok(true) => {} + Ok(false) => return None, + Err(e) => { + tracing::error!("guest session check failed for {w_id}: {e:#}"); + return None; + } + } + } Some(opt_job_authed) } @@ -159,6 +175,18 @@ impl AuthCache { if is_no_auth() { return Some(OptJobAuthed { authed: no_auth_admin_authed(), job_id: None }); } + // Reject an oversized guest bearer before the cache key is built from it: the key + // copies and hashes the whole token, so the cap should bound that work too. Log it + // like the other guest refusals, since get_opt_job_authed turns None into a bare 401. + if token.starts_with(windmill_common::guest_jwt::BEARER_PREFIX) + && token.len() > windmill_common::guest_jwt::MAX_GUEST_JWT_LEN + { + tracing::error!( + "guest JWT refused: bearer is longer than {} bytes", + windmill_common::guest_jwt::MAX_GUEST_JWT_LEN + ); + return None; + } let key = ( w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string(), @@ -200,6 +228,111 @@ impl AuthCache { None } } + _ if token.starts_with(windmill_common::guest_jwt::BEARER_PREFIX) => { + // A workspace-less route never accepts a guest JWT: the identity is + // pinned to the workspace its claim names, like a DB guest session. + let Some(w_id) = w_id.as_deref() else { + return None; + }; + // Strip exactly one prefix: `trim_start_matches` would strip repeated prefixes, + // so `jwt_guest_jwt_guest_` would reduce to a valid token that verifies and + // is then cached under the full, non-canonical bearer key. + let jwt = token + .strip_prefix(windmill_common::guest_jwt::BEARER_PREFIX) + .unwrap_or(token); + let claims = + match windmill_common::guest_jwt::verify_for_workspace(&self.db, w_id, jwt) + .await + { + Ok(c) => c, + Err(e) => { + tracing::error!("guest JWT auth error for {w_id}: {e:#}"); + return None; + } + }; + // The workspace switch, the instance switch and the app being in guest + // mode, in one answer (guest_app_admits). The door re-reads the switches + // and the no-account rule per request through the sentinel below + // (guest_session_stands), so turning any of them off stops a cached JWT + // session on its next call. + match windmill_common::workspaces::guest_app_admits( + &self.db, + w_id, + &claims.app_path, + ) + .await + { + Ok(true) => {} + Ok(false) => return None, + Err(e) => { + tracing::error!("guest JWT admit check failed for {w_id}: {e:#}"); + return None; + } + } + // Resolve on the lowercased email: accounts are stored lowercased, so a + // mixed-case claim would otherwise slip past the no-account gate and + // resolve an account holder to a guest, and split the activity rows the + // seat count reads. + let email = claims.email.to_lowercase(); + // A guest is someone with no account at all; an account holder is refused, + // never downgraded (the same rule as the signed-in guest mint). + match windmill_common::users::has_any_account(&self.db, &email).await { + Ok(false) => {} + Ok(true) => return None, + Err(e) => { + tracing::error!("guest JWT account check failed: {e:#}"); + return None; + } + } + // The instance allowance, checked and recorded transactionally. A stranger + // past the cap on a capped instance is refused here; a returning guest + // always passes. Recording an account holder is avoided by the check above. + if !admit_and_record_guest_jwt(&self.db, w_id, &email, &claims.app_path).await { + return None; + } + // guest_session_scopes already carries the sentinel, and it is the whole + // grant; a JWT has no label, so the sentinel is what governs it. It also + // re-checks the path holds no scope metacharacter (verify already did). + let scopes = match crate::scopes::guest_session_scopes(&claims.app_path) { + Ok(s) => Some(s), + Err(e) => { + tracing::error!("guest JWT app_path cannot be scoped for {w_id}: {e:#}"); + return None; + } + }; + // The JWT's own expiry caps a token minted from this session. The auth + // cache entry itself is capped far shorter (GUEST_JWT_CACHE_TTL) so a + // rotated or cleared key stops the session on re-verification, within + // minutes, rather than only at exp (up to 24h away). + let credential_expiry = + chrono::Utc.timestamp_nanos(claims.exp as i64 * 1_000_000_000); + let cache_expiry = credential_expiry.min(chrono::Utc::now() + GUEST_JWT_CACHE_TTL); + let authed = ApiAuthed { + username: email.clone(), + email, + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes, + username_override: None, + username_override_is_token_label: false, + is_session_token: false, + token_prefix: Some(safe_token_prefix(token)), + read_only: false, + job_id: None, + credential_expiry: Some(credential_expiry), + }; + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: cache_expiry, + job_id: None, + }, + ); + Some(OptJobAuthed { authed, job_id: None }) + } _ if token.starts_with("jwt_") => { let jwt_token = token.trim_start_matches("jwt_"); @@ -233,6 +366,7 @@ impl AuthCache { token_prefix: claims.audit_span, read_only: false, job_id: None, + credential_expiry: None, }; // Fail closed: a `job_id` claim that does not parse must reject // the token rather than resolve to `None`, which would clear the @@ -347,6 +481,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } else { tracing::warn!( @@ -400,6 +535,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } else { tracing::warn!( @@ -427,6 +563,8 @@ impl AuthCache { } (_, Some(email), super_admin, scopes, label, read_only) => { let is_session_token = is_session_label(label.as_deref()); + let is_guest_session = + windmill_common::auth::is_guest_session_label(label.as_deref()); let (username_override, username_override_is_token_label) = username_override_from_label(label); if w_id.is_some() { @@ -476,6 +614,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } None if super_admin => { @@ -500,6 +639,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }), Err(e) => { tracing::error!( @@ -509,6 +649,37 @@ impl AuthCache { } } } + // A guest session: IdP-authenticated, member of + // nothing. No `usr` lookup, groups or folders, so + // every ACL denies it and the token's scopes are + // its whole grant. After the superadmin arm, so + // that token is never demoted into this one. + None if is_guest_session => { + // The server-minted label is the grant, never + // the `guest` scope (a user-minted token's + // scopes are whatever the caller typed); the + // sentinel is pinned on here so every guest + // control downstream sees a guest regardless. + let scopes = Some(crate::scopes::with_guest_sentinel( + scopes.unwrap_or_default(), + )); + Some(ApiAuthed { + username: email.clone(), + email, + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes, + username_override, + username_override_is_token_label, + is_session_token, + token_prefix: Some(safe_token_prefix(token)), + read_only, + job_id: None, + credential_expiry: None, + }) + } None => None, } } else { @@ -526,6 +697,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } } @@ -564,6 +736,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only: false, job_id: None, + credential_expiry: None, }; Some(OptJobAuthed { authed, job_id: None }) } else { @@ -574,6 +747,127 @@ impl AuthCache { } } +/// How long a guest JWT resolves from the auth cache before the arm re-runs (and +/// re-reads the key). A guest JWT is not revocable except by the workspace switch or +/// by rotating the key, so the entry must be short enough that a rotated key bites +/// soon, unlike a normal token whose row can be deleted. Also what makes the +/// day-keyed activity dedupe below reachable across a midnight. +const GUEST_JWT_CACHE_TTL: chrono::Duration = chrono::Duration::minutes(5); + +/// A refused JWT (a stranger past the allowance) is remembered this long so a replayed +/// bearer does not take the instance-wide allowance advisory lock on every request. +/// Short, so a stranger admitted once the window frees is re-checked soon. +const GUEST_JWT_REFUSED_TTL: std::time::Duration = std::time::Duration::from_secs(30); + +lazy_static::lazy_static! { + // One `guest_activity` upsert and one `users.login_guest` audit per email, + // workspace and day: the arm re-runs every GUEST_JWT_CACHE_TTL, and neither the + // seat scan nor the audit trail wants a write each time. LRU-bounded; the day is in + // the key, so a new day writes again. + static ref GUEST_JWT_ACTIVITY_CACHE: Cache = Cache::new(2000); + static ref GUEST_JWT_REFUSED_CACHE: Cache = Cache::new(2000); +} + +/// Admit a JWT guest against the instance allowance and record today's activity, in one +/// transaction so the advisory lock in `guest_admission` spans the count check and the +/// row that changes it. Returns false when the allowance refuses the email or on a DB +/// error, both of which deny the guest. Cached per email, workspace and day: a bearer +/// replayed every request runs this at most once a day, and a refused one is remembered +/// briefly so it does not re-take the allowance lock. `email` is already lowercased. +async fn admit_and_record_guest_jwt(db: &DB, w_id: &str, email: &str, app_path: &str) -> bool { + let cache_key = format!("{email}|{w_id}|{}", chrono::Utc::now().date_naive()); + if GUEST_JWT_ACTIVITY_CACHE.get(&cache_key).is_some() { + return true; + } + if GUEST_JWT_REFUSED_CACHE + .get(&cache_key) + .is_some_and(|at| at.elapsed() < GUEST_JWT_REFUSED_TTL) + { + return false; + } + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("guest JWT tx begin failed for {w_id}: {e:#}"); + return false; + } + }; + // The allowance and the row that changes it, in one transaction: guest_admission + // takes a transaction-scoped advisory lock, so the count check and the insert cannot + // race two strangers past the cap. Only a real allowance refusal is negative-cached; + // a transient DB error denies this request but must not lock the email out for 30s. + match windmill_common::workspaces::guest_admission(&mut *tx, email).await { + Ok(()) => {} + Err(e @ windmill_common::error::Error::PermissionDenied(_)) => { + // The guest hits a bare 401 (the reason must not leak to an unauthenticated caller); + // warn so an admin sees the cap in logs, since it is the actionable signal here. + tracing::warn!("guest JWT refused (guest allowance) for {w_id}: {e:#}"); + GUEST_JWT_REFUSED_CACHE.insert(cache_key, std::time::Instant::now()); + return false; + } + Err(e) => { + tracing::error!("guest JWT allowance check failed for {w_id}: {e:#}"); + return false; + } + } + // The conditional `WHERE NOT jwt_entry` flips the flag only on its false-to-true + // transition, so the upsert returns a row exactly once per email per day: on the + // fresh insert, or on the first JWT after an identity-provider sign-in created + // today's row with `jwt_entry = false`. The audit is gated on that, decided + // atomically by the conflicting tuple, so concurrent first requests (a metered + // instance takes no advisory lock) audit at most once. + let first_jwt = sqlx::query_scalar!( + r#"INSERT INTO guest_activity (email, workspace_id, day, jwt_entry) + VALUES ($1, $2, CURRENT_DATE, true) + ON CONFLICT (email, workspace_id, day) + DO UPDATE SET jwt_entry = true, last_seen_at = now() + WHERE NOT guest_activity.jwt_entry + RETURNING 1 AS "audited!""#, + email, + w_id, + ) + .fetch_optional(&mut *tx) + .await; + let first_jwt = match first_jwt { + Ok(v) => v.is_some(), + Err(e) => { + tracing::error!("recording guest JWT activity for {w_id}: {e:#}"); + return false; + } + }; + if let Err(e) = tx.commit().await { + tracing::error!("guest JWT tx commit failed for {w_id}: {e:#}"); + return false; + } + GUEST_JWT_ACTIVITY_CACHE.insert(cache_key, ()); + // Audit last, best-effort, on its own connection: the EE writer swallows an + // `audit_partitioned` failure but that failing statement still aborts the + // transaction it runs in, so auditing before the commit would let the whole + // activity row roll back while this returned success, admitting an uncounted guest. + if first_jwt { + let author = windmill_common::audit::AuditAuthor { + email: email.to_string(), + username: email.to_string(), + username_override: None, + token_prefix: None, + }; + if let Err(e) = windmill_audit::audit_oss::audit_log( + db, + &author, + "users.login_guest", + windmill_audit::ActionKind::Create, + w_id, + Some(app_path), + Some([("entry", "jwt")].into()), + ) + .await + { + tracing::error!("auditing guest JWT login for {w_id}: {e:#}"); + } + } + true +} + pub(crate) async fn extract_token(parts: &mut Parts, state: &S) -> Option { let auth_header = parts .headers @@ -774,6 +1068,7 @@ fn no_auth_admin_authed() -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -934,10 +1229,17 @@ pub(crate) fn username_override_from_label(label: Option) -> (Option ( - Some(format!("{}{label}", crate::GENERIC_TOKEN_LABEL_PREFIX)), - true, - ), + Some(label) + if label != "ephemeral-script" + && label != "session" + && label != windmill_common::auth::GUEST_SESSION_LABEL + && !label.is_empty() => + { + ( + Some(format!("{}{label}", crate::GENERIC_TOKEN_LABEL_PREFIX)), + true, + ) + } _ => (None, false), } } diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index f2392839ba..04513b4c05 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -78,6 +78,11 @@ pub struct ApiAuthed { /// member can point at a superadmin, so it must never be trusted as a global /// superadmin (`require_super_admin`), GHSA-hfh4-cx4h-3fcr. pub job_id: Option, + /// When this credential itself expires, if it carries its own expiry rather than a + /// token row. Set for a guest JWT (its `exp`): a token minted from it is capped at + /// this, since the JWT's expiry is a guest's only revocation and there is no row to + /// look the limit up in. `None` for every credential whose limit lives in `token`. + pub credential_expiry: Option>, } impl ApiAuthed { @@ -165,6 +170,7 @@ impl From for ApiAuthed { token_prefix: value.token_prefix, read_only: false, job_id: None, + credential_expiry: None, } } } @@ -1047,6 +1053,10 @@ pub async fn fetch_api_authed_from_permissioned_as( db: &DB, username_override: Option, ) -> error::Result { + // Keyed by the supplied address, so an entry built for a principal's previous holder is reused + // while that address is still supplied, until its 120s expiry: a cached dispatch address is + // evicted sooner, an app's stored one (a username deleted then reused) may not be. Accepted; + // the rebuild after expiry is the current holder's. let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); let mut api_authed = match API_AUTHED_CACHE.get(&key) { @@ -1062,7 +1072,10 @@ pub async fn fetch_api_authed_from_permissioned_as( let api_authed = ApiAuthed { username: authed.username, - email, + // The resolved one, not the address we were handed: that is the point of + // `fetch_authed_from_permissioned_as` validating it against the principal's live + // binding, and this value goes on to the job row, `job_perms` and the JWT. + email: authed.email, is_admin: authed.is_admin, is_operator: authed.is_operator, groups: authed.groups, @@ -1074,6 +1087,7 @@ pub async fn fetch_api_authed_from_permissioned_as( token_prefix: authed.token_prefix, read_only: false, job_id: None, + credential_expiry: None, }; API_AUTHED_CACHE.insert( diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 05de1183cd..29ee0fe36a 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -500,6 +500,24 @@ pub fn check_route_access( } } + // A guest session carries the same broad read scopes as an embed token and for + // the same handful of routes, so it gets the same default-deny. + if has_guest_sentinel(Some(token_scopes)) { + if let Some(suffix) = route_suffix.as_deref() { + if guest_route_denied(required_domain, suffix) { + return Err(Error::PermissionDenied(format!( + "a guest session cannot access {route_path}" + ))); + } + // Same rationale as the embed branch: re-running a component supersedes + // its in-flight run, and `cancel_job_api` confines this to the caller's + // own jobs. + if suffix.starts_with("jobs_u/queue/cancel/") { + return Ok(()); + } + } + } + // Each declared scope must grant what its prompt said and no more: // `jobs:run` only deployed runnables, `users:read` only the viewer's identity. if has_raw_app_sdk_sentinel(Some(token_scopes)) { @@ -753,6 +771,55 @@ pub fn has_app_embed_sentinel(scopes: Option<&[String]>) -> bool { scopes.is_some_and(|s| s.iter().any(|x| x == APP_EMBED_SENTINEL)) } +/// Sentinel in a guest session token: someone the identity provider authenticated +/// who is a member of no workspace. Grants nothing itself — it only confines the +/// session to the app surface, the same way `app_embed` does. What makes a session a +/// guest at all is the server-minted label +/// [`windmill_common::auth::GUEST_SESSION_LABEL`]; a forged sentinel here can only +/// narrow its own token. +pub const GUEST_SENTINEL: &str = "guest"; + +/// True if a token is a guest session, whose scopes are its entire grant: it has no ACL +/// of its own, so every ACL check denies it unaided. +pub fn has_guest_sentinel(scopes: Option<&[String]>) -> bool { + scopes.is_some_and(|s| s.iter().any(|x| x == GUEST_SENTINEL)) +} + +/// `scopes` with the guest sentinel present exactly once. +pub fn with_guest_sentinel(mut scopes: Vec) -> Vec { + if !scopes.iter().any(|x| x == GUEST_SENTINEL) { + scopes.push(GUEST_SENTINEL.to_string()); + } + scopes +} + +/// Scopes a guest session carries. The broad-looking reads are narrowed to a route +/// allowlist by the sentinel (`guest_route_denied`), plus the two path-scoped app +/// grants. A guest has no `usr` row, so this list is the whole of what it can do. The +/// single source both the mint (a signed-in guest) and the JWT auth arm build from. +/// +/// The sentinel here only narrows. A signed-in guest is made one by the server-minted +/// label; a JWT guest has no label, so for it the sentinel is what governs. +pub fn guest_session_scopes(app_path: &str) -> windmill_common::error::Result> { + // The path is spliced into a scope, whose grammar reserves `:`, `,`, `*` and a leading + // `/`; app paths may otherwise carry spaces and `@`, so guard only those reserved chars. + if !windmill_common::auth::is_scope_literal_path(app_path) { + return Err(windmill_common::error::Error::BadRequest(format!( + "app path {app_path} is empty or cannot be scoped: `:`, `,` and `*` are reserved \ + in scopes, and a leading `/` never matches a route" + ))); + } + Ok(vec![ + GUEST_SENTINEL.to_string(), + "jobs:read".to_string(), + "resources:run".to_string(), + "users:read".to_string(), + "folders:read".to_string(), + format!("apps:read:{app_path}"), + format!("apps:run:{app_path}"), + ]) +} + /// Sentinel in raw-app SDK tokens. Grants nothing; `check_route_access` uses it /// to narrow the declared scopes to what the viewer's prompt promised. pub const RAW_APP_SDK_SENTINEL: &str = "raw_app_sdk"; @@ -815,6 +882,19 @@ fn app_embed_apps_route_allowed(suffix: &str) -> bool { suffix.starts_with("apps/get/p/") || suffix.starts_with("apps_u/") } +/// Routes a guest session is denied: the app-embed allowlist, plus the embed-token +/// mint. A guest session is the *embedder* — the viewer's own browser rendering the +/// app page — not the app's own JS, and the page mints the iframe's token from it. +/// +/// Everything else stays default-denied, so a guest reaches the app it was let in +/// for and nothing around it. +fn guest_route_denied(domain: ScopeDomain, suffix: &str) -> bool { + if domain == ScopeDomain::Apps && suffix.starts_with("apps_u/embed_token") { + return false; + } + app_embed_route_denied(domain, suffix) +} + /// Job routes a running app uses (the by-id poll/cancel surface driven by the /// frontend JobLoader). Everything else in the jobs domain — enumeration, counts, /// exports, and the `job_signature`/`resume_urls` capability-minting routes — is @@ -993,12 +1073,15 @@ fn scope_grants_access( /// the caller's own row; `email` and `allowed_domain_auto_invite` are derived from the /// token itself and touch no table. /// -/// `settings/global/automate_username_creation` is the one instance setting on the list. -/// `get_global_setting` exempts a handful of keys from its own super-admin gate, that one -/// among them, so the boolean is already readable by every authenticated user; it is here -/// because the CLI reads it before creating a user during a git-sync push, which runs as a -/// job. The other ungated keys have no such caller, so they stay confined — being ungated -/// earns a key nothing on its own. +/// Three instance settings are on the list. `get_global_setting` exempts a handful of keys +/// from its own super-admin gate, these among them, so each is already readable by every +/// authenticated user; each is here because the CLI reads it from a job: +/// `automate_username_creation` before creating a user during a git-sync push, `uid` and +/// `hub_base_url` when `u/admin/hub_sync` pulls resource types from the Hub. The other +/// ungated keys have no such caller, so they stay confined — being ungated earns a key +/// nothing on its own. Listing a gated key earns it nothing either: `require_super_admin` +/// refuses every job token, so `hub_api_secret`, which that pull reads for a private Hub, +/// stays out of a job's reach whatever this list says. /// /// Deliberately absent, as each crosses that line: `users/list_invites` (returns the /// workspace ids the identity was invited to), `users/tokens/list` (credential metadata @@ -1016,6 +1099,8 @@ fn is_global_read_open_to_job_token(route_path: &str) -> bool { | "/api/users/tutorial_progress" | "/api/workspaces/allowed_domain_auto_invite" | "/api/settings/global/automate_username_creation" + | "/api/settings/global/uid" + | "/api/settings/global/hub_base_url" | "/api/docs/search" | "/api/docs/page" | "/api/integrations/hub/list" diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 7ad99996d5..7598021f00 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -59,14 +59,116 @@ struct Config { config: serde_json::Value, } +/// Credential-bearing fields across the `ObjectSettings` variants, which are flattened into a +/// single object by the `type` tag. +const OBJECT_STORE_SECRET_KEYS: &[&str] = + &["access_key", "secret_key", "accessKey", "serviceAccountKey"]; + +/// What an obfuscated read shows a caller who may not see the real credential. +const OBJECT_STORE_SECRET_MASK: &str = "*****"; + +/// Blank the secrets in one worker-group config, in place. +/// +/// Worker-group configs are instance-global and expose `env_vars_static` and the bucket +/// credentials of `object_store_cache_config`; a job token (capped at workspace admin) gets this +/// view even when its identity is a superadmin, as does a devops user who is not an instance +/// admin. See `is_instance_admin` (GHSA-hfh4-cx4h-3fcr). Every route that returns a worker-group +/// config must go through here — a single unobfuscated read hands over the whole bucket. +fn obfuscate_worker_config(config: &mut serde_json::Value) { + let Some(config) = config.as_object_mut() else { + return; + }; + if let Some(env_vars) = config + .get_mut("env_vars_static") + .and_then(|v| v.as_object_mut()) + { + for (_, value) in env_vars.iter_mut() { + // the value is a string, so to_string() it and take -2 to drop the quotes + *value = serde_json::json!("*".repeat(value.to_string().len().saturating_sub(2))); + } + } + if let Some(store) = config + .get_mut("object_store_cache_config") + .and_then(|v| v.as_object_mut()) + { + for key in OBJECT_STORE_SECRET_KEYS { + if let Some(secret) = store.get_mut(*key) { + *secret = serde_json::json!(OBJECT_STORE_SECRET_MASK); + } + } + } +} + +/// Put back the credentials behind [`OBJECT_STORE_SECRET_MASK`]. A devops user who is not an +/// instance admin edits the group from the obfuscated view, so a plain save would otherwise +/// store the mask as the secret and take the group's dependency cache offline — silently, since +/// a worker that cannot build its override just falls back to caching on local disk. +async fn restore_masked_object_store_secrets( + db: &DB, + name: &str, + config: &mut serde_json::Value, +) -> error::Result<()> { + if !has_masked_object_store_secret(config) { + return Ok(()); + } + let stored = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name = $1", + name + ) + .fetch_optional(db) + .await? + .map(|c| c.config); + restore_object_store_secrets(config, stored.as_ref()); + Ok(()) +} + +fn has_masked_object_store_secret(config: &serde_json::Value) -> bool { + let Some(store) = config.get("object_store_cache_config") else { + return false; + }; + OBJECT_STORE_SECRET_KEYS + .iter() + .any(|k| store.get(k).and_then(|v| v.as_str()) == Some(OBJECT_STORE_SECRET_MASK)) +} + +/// The half of [`restore_masked_object_store_secrets`] after the read. +fn restore_object_store_secrets( + config: &mut serde_json::Value, + stored: Option<&serde_json::Value>, +) { + let stored = stored + .and_then(|c| c.get("object_store_cache_config")) + .and_then(|v| v.as_object()) + .cloned(); + let Some(store) = config + .get_mut("object_store_cache_config") + .and_then(|v| v.as_object_mut()) + else { + return; + }; + for key in OBJECT_STORE_SECRET_KEYS { + if store.get(*key).and_then(|v| v.as_str()) != Some(OBJECT_STORE_SECRET_MASK) { + continue; + } + match stored.as_ref().and_then(|s| s.get(*key)) { + Some(secret) => store.insert(key.to_string(), secret.clone()), + // Nothing to put back: drop the mask rather than store it. + None => store.remove(*key), + }; + } +} + async fn list_worker_groups( authed: ApiAuthed, Extension(db): Extension, ) -> error::JsonResult> { - let mut configs_raw = - sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name LIKE 'worker__%'") - .fetch_all(&db) - .await?; + let mut configs_raw = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name LIKE 'worker__%'" + ) + .fetch_all(&db) + .await?; // Remove the 'worker__' prefix from all config names for config in configs_raw.iter_mut() { if let Some(name) = &config.name { @@ -75,44 +177,12 @@ async fn list_worker_groups( } } } - // Worker-group configs are instance-global and expose env_vars_static (may hold - // secrets); a job token (capped at workspace admin) gets the obfuscated view even - // when its identity is a superadmin. See is_instance_admin (GHSA-hfh4-cx4h-3fcr). - let configs = if !is_instance_admin(&authed) { - let mut obfuscated_configs: Vec = vec![]; - for config in configs_raw { - let config_value_opt = config.config.as_object().map(|obj| obj.to_owned()); - if let Some(mut config_value) = config_value_opt { - if let Some(env_var_map) = config_value - .get("env_vars_static") - .map(|obj| obj.as_object()) - .flatten() - { - let mut new_env_var_map: serde_json::Map = - serde_json::Map::new(); - for (key, value) in env_var_map { - new_env_var_map.insert( - key.to_owned(), - // we know the value is a string here, so we to_string() it and take -2 to remove the quotes - serde_json::json!("*".repeat(value.to_string().len() - 2)), - ); - } - config_value.insert( - "env_vars_static".to_string(), - serde_json::Value::Object(new_env_var_map), - ); - } - obfuscated_configs.push(Config { - name: config.name, - config: serde_json::Value::Object(config_value), - }) - } + if !is_instance_admin(&authed) { + for config in configs_raw.iter_mut() { + obfuscate_worker_config(&mut config.config); } - obfuscated_configs - } else { - configs_raw - }; - Ok(Json(configs)) + } + Ok(Json(configs_raw)) } async fn get_config( @@ -122,10 +192,20 @@ async fn get_config( ) -> error::JsonResult> { require_devops_role(&db, &authed).await?; - let config = sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name = $1", name) - .fetch_optional(&db) - .await? - .map(|c| c.config); + let mut config = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name = $1", + name + ) + .fetch_optional(&db) + .await? + .map(|c| c.config); + + if !is_instance_admin(&authed) { + if let Some(config) = config.as_mut() { + obfuscate_worker_config(config); + } + } Ok(Json(config)) } @@ -134,10 +214,14 @@ async fn update_config( Path(name): Path, Extension(db): Extension, authed: ApiAuthed, - Json(config): Json, + Json(mut config): Json, ) -> error::Result { require_devops_role(&db, &authed).await?; + if name.starts_with("worker__") { + restore_masked_object_store_secrets(&db, &name, &mut config).await?; + } + #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode @@ -321,9 +405,14 @@ async fn list_configs( Extension(db): Extension, ) -> error::JsonResult> { require_devops_role(&db, &authed).await?; - let configs = sqlx::query_as!(Config, "SELECT name, config FROM config") + let mut configs = sqlx::query_as!(Config, "SELECT name, config FROM config") .fetch_all(&db) .await?; + if !is_instance_admin(&authed) { + for config in configs.iter_mut() { + obfuscate_worker_config(&mut config.config); + } + } Ok(Json(configs)) } @@ -414,3 +503,49 @@ async fn list_all_dedicated_with_deps( Ok(Json(result)) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The mask an obfuscated read hands out must never be storable as the credential itself: + /// a devops user who is not an instance admin edits the group from that view, and a worker + /// that cannot build its override degrades to a local-only cache without failing a job, so + /// the breakage would go unnoticed. + #[test] + fn masked_secrets_survive_a_save_from_the_obfuscated_view() { + let stored = serde_json::json!({ + "object_store_cache_config": { + "type": "S3", "bucket": "cache", "access_key": "AKIA", "secret_key": "s3cr3t" + }, + "env_vars_static": { "TOKEN": "hunter2" }, + }); + + let mut shown = stored.clone(); + obfuscate_worker_config(&mut shown); + let store = &shown["object_store_cache_config"]; + assert_eq!(store["secret_key"], OBJECT_STORE_SECRET_MASK); + assert_eq!(store["access_key"], OBJECT_STORE_SECRET_MASK); + assert_eq!(store["bucket"], "cache"); + assert_ne!(shown["env_vars_static"]["TOKEN"], "hunter2"); + + let mut saved = shown.clone(); + saved["object_store_cache_config"]["bucket"] = serde_json::json!("other"); + restore_object_store_secrets(&mut saved, Some(&stored)); + let store = &saved["object_store_cache_config"]; + assert_eq!(store["secret_key"], "s3cr3t"); + assert_eq!(store["access_key"], "AKIA"); + assert_eq!(store["bucket"], "other"); + } + + #[test] + fn a_mask_with_nothing_behind_it_is_dropped_rather_than_stored() { + let mut saved = serde_json::json!({ + "object_store_cache_config": { "type": "S3", "secret_key": OBJECT_STORE_SECRET_MASK } + }); + restore_object_store_secrets(&mut saved, None); + assert!(saved["object_store_cache_config"] + .get("secret_key") + .is_none()); + } +} diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 4170c46392..cf55a07078 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -827,6 +827,29 @@ async fn create_flow( WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() }, ); + // Trigger CI tests for items that reference this flow + { + let db2 = db.clone(); + let w_id2 = w_id.clone(); + let flow_path2 = nf.path.clone(); + let email2 = authed.email.clone(); + let username2 = authed.username.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_dep_map::ci_tests::trigger_ci_tests_for_item( + &db2, + &w_id2, + &flow_path2, + "flow", + &email2, + &username2, + ) + .await + { + tracing::error!(%e, "error triggering CI tests after flow creation"); + } + }); + } + Ok((StatusCode::CREATED, nf.path.to_string())) } @@ -926,7 +949,7 @@ async fn derived_on_behalf_of_email( let Some(permissioned_as) = flow.on_behalf_of.as_deref() else { return Ok(None); }; - // Uncached, for the reason given on `prefetch_cached_script`: this pair is round-tripped. + // Uncached: this pair is round-tripped by the client and stored again on redeploy. Ok(Some( windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) .await?, diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index c6f88dace5..07b8004d83 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -318,6 +318,19 @@ async fn add_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } @@ -528,6 +541,19 @@ async fn remove_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } } diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs new file mode 100644 index 0000000000..60f846ca1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response { + authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap() +} + +/// A `dbt://` relation is one graph node only while every side spells it the same +/// way, and three sides derive that spelling independently: the `// materialize` +/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and +/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer` +/// prove the predicate; only a deploy proves the handler feeds it the key the +/// table actually holds — so a canonicalization that drifted on one side would +/// pass those and split the node here. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + sqlx::query!( + r#"UPDATE workspace_settings + SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb + WHERE workspace_id = 'test-workspace'"# + ) + .execute(&db) + .await?; + + // Nothing generates warehouse DDL, so a managed target is refused rather than + // degraded into the track-only mode it would silently become. + let resp = deploy( + port, + "u/test-user/managed", + "// materialize dbt://main/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("must be `manual`")); + + // The warehouse segment is the identity a dbt model keys on; a name the + // workspace does not configure strands the write on an unreachable node. + let resp = deploy( + port, + "u/test-user/unknown_wh", + "// materialize manual dbt://nope/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("does not configure")); + + // Only the DuckDB executor runs `// data_test` probes, and it runs them around + // a managed write — so a declarer in another language would deploy green with + // its assertions silently never executed. + let resp = deploy( + port, + "u/test-user/tested", + "// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp + .text() + .await? + .contains("`// data_test` is not supported")); + + // Both halves are held to the same relation: every producer is a whole + // `//` under a configured warehouse, so a + // subscription to anything else names something nothing can ever write. + for (path, ref_, expected) in [ + ( + "u/test-user/partial_sub", + "dbt://main/analytics", + "not a whole warehouse relation", + ), + ( + "u/test-user/unknown_wh_sub", + "dbt://nope/analytics/orders", + "does not configure", + ), + ] { + let resp = deploy( + port, + path, + &format!("// on {ref_}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains(expected)); + } + + // Past `asset.path`'s column, where the manifest ingest drops the relation and + // no producer row can exist on either side — computed from the bound so it + // cannot drift under it. + let overlong = format!( + "main/analytics/{}", + "o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN) + ); + let resp = deploy( + port, + "u/test-user/overlong_sub", + &format!("// on dbt://{overlong}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("characters an asset path holds")); + + // Any language may declare the write — the DuckLake write engine is DuckDB's, + // this declaration is not — and the target is canonicalized on the way into + // `asset`, so a hand-written mixed-case spelling lands on the model's key. + let resp = deploy( + port, + "u/test-user/ingest", + "// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + // The create response, not `{:x}` over the stored i64: `ScriptHash` decodes + // hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash + // under 2^60 would 422 the rename below instead of reaching the refusal. + let ingest_hash = resp.text().await?; + let write = sqlx::query_scalar!( + "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \ + AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'" + ) + .fetch_one(&db) + .await?; + assert_eq!(write, "main/analytics/orders"); + + // That producer is native, so subscribing to what it writes is accepted — and + // the `// on` ref has to canonicalize identically, or the row it stores names + // a relation nothing produces. + let resp = deploy( + port, + "u/test-user/consumer", + "// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + let trigger_ref = sqlx::query_scalar!( + "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \ + AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'" + ) + .fetch_one(&db) + .await?; + assert_eq!(trigger_ref, "dbt://main/analytics/orders"); + + // With dbt as the only producer the same subscription can never be woken — a + // dbt run does not dispatch — so the deploy refuses it and names the project. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = deploy( + port, + "u/test-user/mart_consumer", + "// on dbt://main/analytics/MARTS\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // A rename is the other half of that: the producer's write still sits at the + // OLD path in the committed snapshot this deploy reads, while the same + // transaction removes it — so it must not count as the producer that would + // wake the subscription the rename adds. + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/ingest_renamed", + "parent_hash": ingest_hash, + "summary": "", + "description": "", + "content": "// on dbt://main/analytics/orders\nexport async function main() {}", + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs index 3e1035ff82..25942e845c 100644 --- a/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs +++ b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs @@ -15,6 +15,12 @@ use windmill_test_utils::*; const SCRIPT_PATH: &str = "u/test-user/mcp_hdr_probe"; +/// A bun lock the executor accepts without installing anything: no dependencies +/// in the `package.json` half, `` for the `bun.lock` half. The empty +/// string is not a substitute: a lock carrying no `//bun.lock` separator is +/// rejected at run time. +const EMPTY_BUN_LOCK: &str = "{}\n//bun.lock\n"; + /// Echoes the two halves of the event separately, so the assertions can tell /// which one a value arrived in. const PREPROCESSOR_SCRIPT: &str = r#" @@ -84,7 +90,7 @@ async fn test_mcp_preprocessor_receives_the_callers_headers( "description": "", "content": PREPROCESSOR_SCRIPT, "language": "bun", - "lock": "", + "lock": EMPTY_BUN_LOCK, "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", @@ -101,13 +107,14 @@ async fn test_mcp_preprocessor_receives_the_callers_headers( resp.text().await.unwrap_or_default() ); - // A script counts as deployed once it has a lock, which normally arrives from - // a dependency job. Planting an empty one keeps the test to the path under - // test instead of a bun resolution whose timing it does not control. - sqlx::query("UPDATE script SET lock = '' WHERE path = $1 AND workspace_id = 'test-workspace'") - .bind(SCRIPT_PATH) - .execute(&db) - .await?; + // A supplied lock queues no dependency job, so the version is deployed (hence + // listable and runnable) as soon as the create returns. + let queued: i64 = sqlx::query_scalar( + "SELECT count(*) FROM v2_job_queue WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(queued, 0, "the supplied lock must queue no dependency job"); let tools = mcp_post( port, diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index fe80e5fdf4..059b3aa2d2 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -18,15 +18,16 @@ use windmill_common::{ variables::{build_crypt, encrypt}, }; use windmill_native_triggers::{ - classify_read_failure, decrypt_oauth_data, delete_native_trigger, - delete_workspace_integration, get_workspace_integration, + classify_read_failure, decrypt_oauth_data, delete_native_trigger, delete_workspace_integration, + get_workspace_integration, github::GitHub, google::{parse_stop_channel_params, should_renew_channel}, - http_error_status, list_native_triggers, map_external_error, + grant_refused, http_error_status, list_native_triggers, map_external_error, + native_trigger_is_enabled, nextcloud::NextCloud, - grant_refused, require_native_integration_use, store_native_trigger, - store_workspace_integration, External, ExternalReadFailure, HttpRequestError, - NativeTriggerConfig, OAuthConfig, ServiceName, + require_native_integration_use, set_native_trigger_enabled, store_native_trigger, + store_workspace_integration, update_native_trigger, External, ExternalReadFailure, + HttpRequestError, NativeTriggerConfig, OAuthConfig, ServiceName, }; // ============================================================================ @@ -63,6 +64,7 @@ fn test_authed() -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -455,6 +457,7 @@ async fn test_delete_integration_full_cascade(db: Pool) -> anyhow::Res &trigger_config, json!({"triggerType": "drive"}), None, + true, ) .await?; @@ -545,6 +548,7 @@ async fn test_cleanup_preserves_triggers(db: Pool) -> anyhow::Result<( &trigger_config, json!({"triggerType": "drive"}), None, + true, ) .await?; @@ -602,6 +606,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< }, json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), None, + true, ) .await?; // An unrelated trigger already sitting on the target path must not be reported as moved. @@ -618,6 +623,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< }, json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), None, + true, ) .await?; @@ -783,7 +789,10 @@ fn test_refresh_failures_blame_only_the_grant_they_refuse() { let ok = Some(StatusCode::OK); assert!(grant_refused(ok, r#"{"error":"bad_refresh_token"}"#)); assert!(grant_refused(ok, r#"{"error":"invalid_grant"}"#)); - assert!(!grant_refused(ok, r#"{"access_token":"t","token_type":"bearer"}"#)); + assert!(!grant_refused( + ok, + r#"{"access_token":"t","token_type":"bearer"}"# + )); } /// A service that is busy or broken has not refused anything, and callers react differently to @@ -825,7 +834,9 @@ fn test_transient_service_failures_are_not_refusals() { body: r#"{"message":"Must have admin rights to Repository."}"#.to_string(), }); assert!( - map_external_error(refused).to_string().contains("admin rights"), + map_external_error(refused) + .to_string() + .contains("admin rights"), "a real 403 keeps its guidance" ); } @@ -849,3 +860,115 @@ fn test_only_service_failures_degrade_the_read() { "a non-provider error must pass through unmapped" ); } + +/// The pause switch a webhook delivery is gated on. A trigger arrives enabled, survives an +/// unrelated edit, and an unknown one reads as enabled so a delivery Windmill cannot place is +/// never silently dropped. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_native_trigger_enabled_toggle(db: Pool) -> anyhow::Result<()> { + insert_test_script(&db, "f/test/handler").await?; + let config = NativeTriggerConfig { + script_path: "f/test/handler".to_string(), + is_flow: false, + webhook_token: "abcdefghij1234567890".to_string(), + }; + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + &config, + json!({"event": "OCA\\Files\\Event\\LoadAdditionalScriptsEvent"}), + None, + true, + ) + .await?; + + assert!( + native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await?, + "a new trigger fires" + ); + + assert!( + set_native_trigger_enabled( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + false + ) + .await? + ); + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await? + ); + + // Saving a configuration must not resume a trigger someone paused. + update_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + &config, + None, + Some("edited"), + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await?, + "an edit leaves the pause in place" + ); + + // A recreate registers a fresh trigger and must be able to come up already paused, in one + // write, rather than being enabled for as long as it takes a second call to arrive. + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-2", + &config, + json!({}), + None, + false, + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-2").await? + ); + + // The conflict branch is a re-registration of a trigger that already exists, so it carries no + // opinion about the pause. + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-2", + &config, + json!({}), + None, + true, + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-2").await?, + "re-registering leaves the pause in place" + ); + + assert!( + !set_native_trigger_enabled( + &db, + "test-workspace", + ServiceName::Nextcloud, + "unknown", + true + ) + .await?, + "nothing to toggle" + ); + assert!( + native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "unknown").await?, + "a trigger Windmill has no row for is not treated as paused" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/offboarding.rs b/backend/windmill-api-integration-tests/tests/offboarding.rs index 4434e6a2b5..7bdeab924b 100644 --- a/backend/windmill-api-integration-tests/tests/offboarding.rs +++ b/backend/windmill-api-integration-tests/tests/offboarding.rs @@ -599,3 +599,59 @@ async fn test_offboard_invalid_target(db: Pool) -> anyhow::Result<()> Ok(()) } + +/// A legacy member named `group-ops` canonicalizes to `g/ops`, the principal the real `ops` group +/// runs as. Offboarding the member must not hand the group's runnables to the replacement. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_offboard_group_prefixed_member_keeps_group_identities( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::raw_sql( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('ops-bot@windmill.dev', 'x', 'password', false, true, 'Ops bot', 'group-ops'); + INSERT INTO usr(workspace_id, email, username, is_admin, role) + VALUES ('test-workspace', 'ops-bot@windmill.dev', 'group-ops', false, 'User'); + INSERT INTO group_(workspace_id, name, summary) VALUES ('test-workspace', 'ops', ''); + INSERT INTO app(workspace_id, path, summary, policy, versions, extra_perms) + VALUES ('test-workspace', 'f/shared/ops_app', '', + '{\"execution_mode\": \"publisher\", \"on_behalf_of\": \"g/ops\", + \"on_behalf_of_email\": \"group-ops@windmill.dev\"}', '{}', '{}');", + ) + .execute(&db) + .await?; + + let preview: serde_json::Value = + authed(client().get(ws_url(port, "offboard_preview/group-ops"))) + .send() + .await? + .json() + .await?; + assert!( + preview["executing_on_behalf"]["apps"].is_null(), + "the group's apps are not the member's to reassign: {preview}" + ); + + let resp = authed(client().post(ws_url(port, "offboard/group-ops"))) + .json(&json!({ + "reassign_to": "u/test-user", + "new_on_behalf_of_user": "test-user", + "delete_user": false + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let principal: Option = sqlx::query_scalar( + "SELECT policy->>'on_behalf_of' FROM app + WHERE workspace_id = 'test-workspace' AND path = 'f/shared/ops_app'", + ) + .fetch_one(&db) + .await?; + assert_eq!(principal.as_deref(), Some("g/ops")); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/protection_rules.rs b/backend/windmill-api-integration-tests/tests/protection_rules.rs index 9469554999..e032acef7b 100644 --- a/backend/windmill-api-integration-tests/tests/protection_rules.rs +++ b/backend/windmill-api-integration-tests/tests/protection_rules.rs @@ -123,6 +123,72 @@ async fn test_protection_rules(db: Pool) -> anyhow::Result<()> { .await?; assert!(!resp.status().is_success(), "Non-admin should be blocked from flows: {}", resp.status()); + // ======================================== + // 4b. ...but a draft-only resource stays deletable: nothing is deployed at + // its path, so its DELETE is a draft discard rather than a deployment. + // ======================================== + + let draft_only_path = "u/test-user-2/draft_only_resource"; + let resp = authed( + client().post(format!("{base}/drafts/update/resource/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ "value": { + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + }})) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Non-admin should save a draft: {}", + resp.text().await? + ); + + let resp = authed( + client().delete(format!("{base}/resources/delete/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Draft-only delete should not be gated by the deploy rules: {}", + resp.text().await? + ); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'resource'::DRAFT_KIND", + ) + .bind(draft_only_path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + // The gate itself is still there for a DEPLOYED resource at the same path. + let resp = authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "Non-admin should still be blocked from creating a resource: {}", + resp.status() + ); + // ======================================== // 5. Admin bypasses protection rule // ======================================== diff --git a/backend/windmill-api-integration-tests/tests/schedules.rs b/backend/windmill-api-integration-tests/tests/schedules.rs index 89ba0ec2cc..881f11ff43 100644 --- a/backend/windmill-api-integration-tests/tests/schedules.rs +++ b/backend/windmill-api-integration-tests/tests/schedules.rs @@ -113,7 +113,9 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { "expected at least 2 schedules, got {}", list.len() ); - assert!(list.iter().any(|s| s["path"] == "u/test-user/test_schedule")); + assert!(list + .iter() + .any(|s| s["path"] == "u/test-user/test_schedule")); // --- list_with_jobs --- let resp = authed(client().get(format!("{base}/list_with_jobs"))) @@ -125,18 +127,14 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert!(!list.is_empty()); // --- update --- - let resp = authed(client().post(schedule_url( - port, - "update", - "u/test-user/test_schedule", - ))) - .json(&json!({ - "schedule": "0 0 */12 * * *", - "timezone": "Europe/Paris" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(schedule_url(port, "update", "u/test-user/test_schedule"))) + .json(&json!({ + "schedule": "0 0 */12 * * *", + "timezone": "Europe/Paris" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); // verify update @@ -204,14 +202,11 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(resp.status(), 200); // --- delete --- - let resp = authed(client().delete(schedule_url( - port, - "delete", - "u/test-user/another_schedule", - ))) - .send() - .await - .unwrap(); + let resp = + authed(client().delete(schedule_url(port, "delete", "u/test-user/another_schedule"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let resp = authed_get(port, "exists", "u/test-user/another_schedule").await; @@ -220,17 +215,108 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Global endpoints ===== // --- preview --- - let resp = authed(client().post(format!( - "http://localhost:{port}/api/schedules/preview" - ))) - .json(&json!({ - "schedule": "0 0 */6 * * *", - "timezone": "UTC" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(format!("http://localhost:{port}/api/schedules/preview"))) + .json(&json!({ + "schedule": "0 0 */6 * * *", + "timezone": "UTC" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "preview: {}", resp.text().await?); Ok(()) } + +/// A schedule with no `schedule` row is listed from the `draft` table, so its +/// DELETE drops that draft, then 404s once nothing is left at the path. A legacy +/// (`email IS NULL`) draft is owned by nobody and stays put. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_draft_only_schedule(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/draft_only_schedule"; + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/trigger_schedule/{path}" + ))) + .json(&json!({ "value": { + "path": path, + "schedule": "0 0 */6 * * *", + "timezone": "UTC", + "script_path": "u/test-user/never_deployed", + "is_flow": false, + }})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "save draft: {}", resp.text().await?); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/schedules/list?include_draft_only=true" + ))) + .send() + .await + .unwrap(); + let listed: Vec = resp.json().await?; + assert!( + listed + .iter() + .any(|s| s["path"] == path && s["draft_only"] == json!(true)), + "draft-only schedule should be listed: {listed:?}" + ); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "nothing left at the path"); + + // A legacy (email IS NULL) draft is owned by nobody and keeps the write gate + // on the drafts routes, so this one must not become a second door to it. + let legacy_path = "u/test-user/legacy_draft_only_schedule"; + sqlx::query( + "INSERT INTO draft (workspace_id, email, path, typ, value) \ + VALUES ('test-workspace', NULL, $1, 'trigger_schedule'::DRAFT_KIND, '{}'::json)", + ) + .bind(legacy_path) + .execute(&db) + .await?; + + let resp = authed(client().delete(schedule_url(port, "delete", legacy_path))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 404, + "legacy draft is not this route's to delete" + ); + + let legacy_remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(legacy_path) + .fetch_one(&db) + .await?; + assert_eq!(legacy_remaining, 1, "the legacy draft should survive"); + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index 835c4c7e34..857b2e02e2 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -585,6 +585,8 @@ pub fn list_completed_jobs_query( let mut sqlb = SqlBuilder::select_from("v2_job_completed") .fields(fields) .order_by( + // The runs page picks its pagination cursor column from this same rule + // (frontend/src/lib/components/runs/useJobsLoader.svelte.ts); change both together. if lq.completed_before.is_some() || lq.completed_after.is_some() || lq.success == Some(false) diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 6ae902e984..d581608c10 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -31,8 +31,8 @@ use windmill_common::{ self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, }, user_drafts::{ - delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, - UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, fetch_draft_only_list_rows, + overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, }, utils::{ escape_ilike_pattern, not_found_if_none, paginate, Pagination, ScheduleType, StripPath, @@ -332,7 +332,7 @@ async fn create_schedule( ) .await?; // email is still written for backwards compat with old workers that don't know about permissioned_as - let resolved_email = windmill_common::users::get_email_from_permissioned_as( + let resolved_email = windmill_common::users::get_email_from_permissioned_as_uncached( &resolved_permissioned_as, &w_id, &db, @@ -545,18 +545,14 @@ async fn edit_schedule( reject_reserved_schedule_path(path)?; let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; - let mut tx = user_db.begin(&authed).await?; // Check schedule for error ScheduleType::from_str(&es.schedule, es.cron_version.as_deref(), true)?; - // Validate dynamic_skip if provided - if let Some(handler_path) = &es.dynamic_skip { - validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; - } - let resolved_edited_by = resolve_edited_by(&authed); + // Resolved on the (non-RLS) pool before the RLS transaction opens: the lookup mid-transaction + // would hold a second connection while `tx` is checked out. let resolved_permissioned_as = resolve_permissioned_as( es.permissioned_as.as_ref(), es.preserve_permissioned_as, @@ -568,7 +564,7 @@ async fn edit_schedule( let resolved_email = if resolved_permissioned_as != windmill_common::users::username_to_permissioned_as(&authed.username) { - windmill_common::users::get_email_from_permissioned_as( + windmill_common::users::get_email_from_permissioned_as_uncached( &resolved_permissioned_as, &w_id, &db, @@ -585,6 +581,13 @@ async fn edit_schedule( Some(&resolved_email), )?; + let mut tx = user_db.begin(&authed).await?; + + // Validate dynamic_skip if provided + if let Some(handler_path) = &es.dynamic_skip { + validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; + } + let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; let schedule = sqlx::query_as!( @@ -1331,6 +1334,18 @@ async fn delete_schedule( .flatten(); if exists.is_none() { + drop(tx); + if delete_draft_only_for_path( + &db, + &w_id, + UserDraftItemKind::TriggerSchedule, + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only schedule {} deleted", path)); + } return Err(windmill_common::error::Error::NotFound(format!( "Schedule {} not found", path diff --git a/backend/windmill-api-scripts/src/asset_inference.rs b/backend/windmill-api-scripts/src/asset_inference.rs index 9c503ae1ae..1c0bc6d57f 100644 --- a/backend/windmill-api-scripts/src/asset_inference.rs +++ b/backend/windmill-api-scripts/src/asset_inference.rs @@ -78,20 +78,26 @@ fn comment_prefix(lang: &ScriptLang) -> Option<&'static str> { | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Nativets - | ScriptLang::Go => Some("//"), + | ScriptLang::Go + | ScriptLang::Php => Some("//"), _ => None, } } /// Mirror of the frontend `parseVolumeAnnotations` (infer.ts): ` /// volume: ` lines in the leading comment block, each an `rw` volume -/// asset. Scanning stops at the first non-comment line (blank lines are -/// skipped), exactly like the frontend. +/// asset. Scanning stops at the first non-comment line; blank lines and PHP's +/// opening tag line are skipped whole (the tag may carry code, so an annotation +/// must sit on its own line below it), exactly like the frontend. fn parse_volume_annotations(content: &str, prefix: &str) -> Vec { let mut volumes = Vec::new(); for line in content.lines() { let trimmed = line.trim(); - if trimmed.is_empty() { + if trimmed.is_empty() + || trimmed + .get(..5) + .is_some_and(|p| p.eq_ignore_ascii_case(" = got.iter().filter(|a| a.kind == AssetKind::Volume).collect(); + assert_eq!(vols.len(), 1); + assert_eq!(vols[0].path, "my_vol"); + } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index c2e9a6f4e7..05e5d5bde5 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1081,6 +1081,57 @@ fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { [(path.to_string(), hash_script(lock))] } +/// The `dbt://` relation both halves of a deploy have to agree on: a whole +/// `//`, under a warehouse this workspace configures. +/// +/// Every producer is held to exactly this — a `// materialize` target here, a +/// descriptor's `profile.warehouse` in the worker — so a subscription to anything +/// else names a relation nothing can ever write. No later deploy fixes that and +/// no dormant-edge warning reports it, since the warning fires on a dbt project's +/// ingest and no project can claim a relation under a warehouse that isn't there. +/// Asking here rather than at each site is what keeps the two from drifting into +/// refusing and accepting the same string. +async fn validate_dbt_relation( + db: &sqlx::Pool, + w_id: &str, + relation: &str, + what: &str, +) -> Result<()> { + if !windmill_parser::asset_parser::is_full_relation_path(relation) { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is not a whole warehouse relation \ + (`dbt:////`)." + ))); + } + // `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that + // outgrows it rather than failing the whole graph, so past the column no + // producer row can exist on either side — a write would be rejected by + // Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text + // that would take the subscription and keep it dormant for good. + let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN; + if relation.chars().count() > max { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is longer than the {max} characters an asset path \ + holds, so it cannot be recorded." + ))); + } + let warehouse = relation.split('/').next().unwrap_or_default(); + // Only the resolver's own "no such warehouse" is the annotation's fault. Its + // other failures — the query, and a setting entry with no `resource_path` — + // keep their own error: blaming the warehouse name for those misdescribes + // them, and flattening the malformed-setting one to a 400 hides a server + // fault behind a client one. + windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse) + .await + .map_err(|e| match e { + Error::NotFound(_) => Error::BadRequest(format!( + "{what} `dbt://{relation}` names a warehouse this workspace does not \ + configure: {e}" + )), + other => other, + }) +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1267,6 +1318,47 @@ async fn create_script_internal<'c>( } } + // A retired path keeps its versions, and the newest is where a redeploy belongs: hashed + // as a first deploy instead, unchanged content lands on the row the path's own first + // version already holds. A deleted version still counts — its row keeps the hash it was + // deployed under even once the content is wiped, so skipping it is what collides. + // + // Any parentless deploy, not only an `auto_parent` one: the CLI names no parent for a + // path its listing no longer shows, which is where a retried push lands. Gated on + // nothing being live there, so a parentless deploy onto a live path still meets the + // path conflict the match below raises. + let mut parent_adopted_from_retired_path = false; + if ns.parent_hash.is_none() && clashing_script.is_none() { + // Locked, not merely read: a competing deploy chaining onto this same candidate + // takes `FOR UPDATE` on it before inserting, so holding the row is what serializes + // the two. Probe first and the child still uncommitted reads as absent. + let candidate = sqlx::query_scalar::<_, i64>( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 \ + ORDER BY created_at DESC LIMIT 1 FOR UPDATE", + ) + .bind(&ns.path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + // Adoptable only if nothing already descends from it: a rename leaves its source + // path holding a version whose child lives at the destination, and a second child + // forks a lineage the guard below requires to be linear. Nothing adoptable means a + // fresh lineage, which has no parent to vary its hash and can still collide. + ns.parent_hash = match candidate { + Some(hash) => sqlx::query_scalar!( + "SELECT 1 FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", + hash, + &w_id + ) + .fetch_optional(&db) + .await? + .is_none() + .then_some(ScriptHash(hash)), + None => None, + }; + parent_adopted_from_retired_path = ns.parent_hash.is_some(); + } + // Must stay below the parent resolution above: an auto_parent deploy hashed before // it carries a first deploy's lineage, so redeploying content the path has held // before collides with that archived version instead of superseding it. The @@ -1311,20 +1403,42 @@ async fn create_script_internal<'c>( )); }; + // Unscoped, and sound only under the lock above: linearity is a property of the + // lineage, not of what this caller may read. A child can sit where they cannot + // see it — a folder they renamed it into, or grants an adopting deploy reset — + // and asked through `tx` it reads as absent, letting the fork through. let clashing_hash_o = sqlx::query_scalar!( "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", p_hash.0, &w_id ) - .fetch_optional(&mut *tx) + .fetch_optional(&db) .await?; if let Some(clashing_hash) = clashing_hash_o { - return Err(Error::BadRequest(format!( - "A script with hash {} with same parent_hash has been found. However, the \ - lineage must be linear: no 2 scripts can have the same parent", - ScriptHash(clashing_hash) - ))); + // Named only when the caller could already read it. The probe above has to be + // unscoped to be correct, but a hash alone reads a script's content back + // through `raw/h/{hash}`, which authorizes nothing per script — so echoing one + // the caller cannot see hands them a way to fetch it. + let visible_to_caller = sqlx::query_scalar!( + "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", + clashing_hash, + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + return Err(Error::BadRequest(if visible_to_caller { + format!( + "A script with hash {} with same parent_hash has been found. However, \ + the lineage must be linear: no 2 scripts can have the same parent", + ScriptHash(clashing_hash) + ) + } else { + "A script with the same parent_hash has been found. However, the lineage \ + must be linear: no 2 scripts can have the same parent" + .to_owned() + })); }; let ScriptWithStarred { script: ps, .. } = @@ -1387,7 +1501,15 @@ async fn create_script_internal<'c>( } Some(_) | None => Ok(Some(ParentInfo { p_hashes: ph, - perms: ps.extra_perms, + // A version adopted above was taken for its lineage, not its grants: a + // retired path may be reused by a different script, which must not start + // life holding an ACL nobody gave it — including one `delete/h` purged. + // A parent the caller named still carries them, as unarchive expects. + perms: if parent_adopted_from_retired_path { + json!({}) + } else { + ps.extra_perms + }, p_path: ps.path, })), }; @@ -1565,34 +1687,90 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// materialize` materializes a `ducklake:///` target from a - // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake - // target would otherwise deploy, register a producer in the asset graph, then - // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), - // and a non-DuckDB script never reaches the executor that records state. The - // managed-only checks (single trailing SELECT, no SQL args) come after — a - // `manual` script owns its DDL and skips them. + // `// materialize` names what this script produces. Two target kinds, and the + // runtime behind each is what constrains the annotation: + // • `ducklake:///
` — the DuckDB executor generates the write + // (or, in `manual` mode, records the state the script wrote itself), so + // the script has to be a DuckDB one and the target has to name a table. + // A non-DuckDB script never reaches that executor. + // • `dbt:////` — a warehouse relation. Nothing + // generates warehouse DDL, so the declaration is track-only (`manual`) + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. + // Any other kind would deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). + // The managed-only checks (single trailing SELECT, no SQL args) come after — + // a `manual` script owns its DDL and skips them. if let Some(m) = pipeline_annotations.materialize.as_ref() { - if ns.language != ScriptLang::DuckDb { - return Err(Error::BadRequest(format!( - "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ - wmll.ducklake helpers to materialize from other languages.", - ns.language.as_str() - ))); - } - if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + use windmill_parser::asset_parser::AssetKind as PAssetKind; + // The producer half of the rule the trigger loop below applies to `// on`: + // a dbt project's writes come from its manifest, and the graph ingest + // republishes this path's asset rows wholesale, so a declared one would be + // wiped by the very deploy that accepted it while its runs kept stamping + // the relation. + if ns.language == ScriptLang::Dbt { return Err(Error::BadRequest( - "`// materialize` only supports a DuckLake target \ - (`ducklake:///
`); other asset kinds aren't materializable." + "a dbt script cannot declare `// materialize`: what a project builds is read \ + from its manifest and published by the graph ingest, not annotated." .to_string(), )); } - if !m.target_path.contains('/') { - return Err(Error::BadRequest(format!( - "`// materialize` needs a table in the target: \ - `ducklake://{0}/
` (got `ducklake://{0}`).", - m.target_path - ))); + match m.target_kind { + PAssetKind::Ducklake => { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages, or declare a \ + warehouse relation with `// materialize manual dbt://…`.", + ns.language.as_str() + ))); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + } + PAssetKind::Dbt => { + // `// data_test` runs as verifier probes the DuckDB executor + // splices around a MANAGED write. Nothing generates a warehouse + // write, so nothing would run them — and unlike the DuckLake + // `manual` case, which at least fails loudly in that executor, a + // declarer in another language would deploy green with its + // data-quality assertions silently never executed. + if !pipeline_annotations.data_tests.is_empty() { + return Err(Error::BadRequest( + "`// data_test` is not supported with a `dbt://` target: the checks run \ + against a managed materialization, and a warehouse relation is \ + written by the script itself. Assert on the relation with a dbt \ + test in the project that reads it." + .to_string(), + )); + } + if !m.manual { + return Err(Error::BadRequest( + "`// materialize dbt://…` must be `manual`: Windmill generates no \ + warehouse DDL, so the script issues its own write and only the outcome \ + is recorded. Write \ + `// materialize manual dbt:////`." + .to_string(), + )); + } + validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target") + .await?; + } + _ => { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake (`ducklake:///
`) or \ + warehouse-relation (`dbt:////`) target; other asset \ + kinds aren't materializable." + .to_string(), + )); + } } if !m.manual { if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { @@ -2361,27 +2539,28 @@ async fn create_script_internal<'c>( // while its own finished runs still render from them. Clearing by path // would empty those run pages for good. if ns.language != ScriptLang::Dbt { - // The saved retry state does go: nothing regenerates it, it is keyed by - // path alone, and it carries one user's failed invocation and its - // arguments. No dbt version is live at this path any more to resume it. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, &ns.path).await?; + // The saved run and environment state do go: nothing regenerates them, + // both are keyed by path alone, and they carry one user's failed + // invocation with its arguments and the project's own manifest. No dbt + // version is live at this path any more to resume or defer to. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, &ns.path).await?; } if let Some(ref old) = p_path_opt { if old != &ns.path { clear_script_triggers(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; clear_static_asset_usage(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; - // The saved retry state travels rather than being cleared: nothing + // The saved state travels rather than being cleared: nothing // regenerates it, so dropping it would throw away a resumable - // failure for what is only a rename. Only while the destination is - // still dbt — a rename that also converts the language would - // otherwise reinstate at the new path the state the branch above - // just cleared, leaving one user's arguments and results under a - // path no dbt script occupies. + // failure and every deferral until the next full run, for what is + // only a rename. Only while the destination is still dbt — a rename + // that also converts the language would otherwise reinstate at the + // new path the state the branch above just cleared, leaving one + // user's arguments and results under a path no dbt script occupies. if ns.language == ScriptLang::Dbt { - windmill_common::dbt_manifest::move_dbt_run_state(&mut tx, &w_id, old, &ns.path) + windmill_common::dbt_manifest::move_dbt_script_state(&mut tx, &w_id, old, &ns.path) .await?; } else { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, old).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, old).await?; } } } @@ -2389,16 +2568,43 @@ async fn create_script_internal<'c>( let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { continue; }; - // A `dbt://` subscription can never fire: dbt is the only producer of a - // warehouse relation (`// materialize` takes DuckLake targets only) and a - // dbt run does not dispatch. Refusing beats persisting a row that draws a - // cascade arrow on the canvas and then never wakes anything. - if trigger_ref.starts_with("dbt://") { - return Err(Error::BadRequest(format!( - "`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \ - runs, and nothing else writes a warehouse relation. Declare the read without \ - `on` to keep the lineage edge, or schedule this script." - ))); + // A `dbt://` subscription fires only when a NON-dbt job materialized the + // relation: `// materialize manual dbt://…` declares such a write, while a + // dbt run records its models and does not dispatch. So refuse exactly the + // edge that cannot fire — one whose relation is already claimed by dbt and + // by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`, + // which takes the workspace pool: under RLS an unreadable native producer + // would refuse a live subscription). + if let Some(relation) = trigger_ref.strip_prefix("dbt://") { + // The subscriber side of the same rule: a dbt project is not woken by + // the asset cascade. Its graph ingest clears these rows for its own + // path, so accepting one here would deploy an edge the dependency job + // then silently removes. + if ns.language == ScriptLang::Dbt { + return Err(Error::BadRequest(format!( + "a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \ + and a project is run on its schedule, not woken by an asset cascade." + ))); + } + validate_dbt_relation(&db, &w_id, relation, "subscription target").await?; + // Both paths under a rename: the old one's committed write row is + // still there and this transaction is about to remove it. + let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) { + Some(old) => vec![ns.path.clone(), old.to_string()], + None => vec![ns.path.clone()], + }; + if let Some(dbt_owner) = + windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths) + .await? + { + return Err(Error::BadRequest(format!( + "`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \ + `{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \ + read without `on` to keep the lineage edge, or schedule this script. A \ + relation written by a `// materialize manual {trigger_ref}` script can be \ + subscribed to." + ))); + } } // Effective debounce for this edge: per-`// on debounce=` wins, // else the script-level `// debounce` default. Debounce only @@ -3594,7 +3800,11 @@ async fn archive_script_by_path( path, &w_id ) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as the by-hash routes are: + // committed on its own, a cleanup that then fails leaves dbt state at a path + // no live version occupies, for whatever is created there next to defer + // through. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; @@ -3602,9 +3812,10 @@ async fn archive_script_by_path( // The graph stays: the pinned read resolves versions through a CTE that // already skips archived rows, so it stops answering for current relations // either way, while deleting it would empty the Models panel of every - // completed run of the project. Retry state does go — nothing may resume a - // script that is no longer live. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + // completed run of the project. The saved run and environment state do go — + // nothing may resume a script that is no longer live, and nothing may defer + // through what it last built. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; // Pipeline event hygiene: an archived script must not be triggered by // anything. Wipe declared `// on ...` edges (asset-event subscribers // look these up). @@ -3689,7 +3900,7 @@ async fn archive_script_by_hash( clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; // The version's graph stays: its finished runs still render from it, and // the live-version CTE already skips archived rows. Deletion clears it. - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3752,7 +3963,12 @@ async fn delete_script_by_hash( ) .bind(&hash.0) .bind(&w_id) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as `archive_script_by_hash` + // already does. Committed on its own, it opens a window where the path has + // no live version and a concurrent deploy can take it — and the retirement + // guard below then finds that new script live, keeps the old project's dbt + // state, and leaves the replacement able to defer through its manifest. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; @@ -3765,7 +3981,7 @@ async fn delete_script_by_hash( windmill_common::dbt_manifest::clear_dbt_manifest_version(&mut tx, &w_id, &script.path, hash.0) .await?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3866,11 +4082,11 @@ async fn delete_script_by_path( // After the DELETE, never before: every dbt writer locks the `script` row // first, so taking a sidecar ahead of it deadlocks one of the pair. The - // VERSIONED graph needs no clear at all, cascading off `script`; the retry - // state does, being keyed by path alone and so inherited by whatever is - // created here next, and so do the editor's own graphs, whose NULL + // VERSIONED graph needs no clear at all, cascading off `script`; the saved + // run and environment state do, being keyed by path alone and so inherited + // by whatever is created here next, and so do the editor's own graphs, whose NULL // `script_hash` satisfies that foreign key without riding its cascade. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, path).await?; if !trash_scripts.is_empty() { @@ -4039,7 +4255,7 @@ async fn delete_scripts_bulk( // Same reason as the single-path delete, over every requested path rather // than the deleted ones: a path that had no script left can still hold state. for p in &request.paths { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, p).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, p).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, p).await?; } diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 18a8d52ac0..0fe358da1c 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -58,9 +58,10 @@ use windmill_common::{ AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, - HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, - RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING, + GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, + RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING, UNIQUE_ID_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WS_BASE_URL_SETTING, @@ -113,6 +114,9 @@ async fn get_ruff_config_unauthed(Extension(db): Extension) -> error::Result pub fn global_service() -> Router { #[warn(unused_mut)] let r = Router::new() + // `/local` is the path in openapi.yaml, so every generated client (getLocal) calls it; + // `/envs` stays for callers that found the route in the code. + .route("/local", get(get_local_settings)) .route("/envs", get(get_local_settings)) .route( "/global/{key}", @@ -1044,6 +1048,12 @@ async fn run_setting_pre_write_hook( } } } + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING => { + // Rejected at write time rather than at boot: a mistyped origin + // matches no request, so it would silently block the very app it + // names with nothing but a log line to go on. + windmill_common::global_settings::parse_allowed_origins_setting(Some(value))?; + } HTTP_ROUTE_WORKSPACED_ROUTE_SETTING => { let serde_json::Value::Bool(workspaced_route) = value else { return Err(error::Error::BadRequest(format!( @@ -1172,6 +1182,18 @@ async fn run_setting_pre_write_hook( } } } + INSTANCE_BANNER_SETTING => { + match value { + // Clearing (delete row) is handled by the caller; allow it through. + serde_json::Value::Null => {} + serde_json::Value::String(s) if s.trim().is_empty() => {} + v => { + windmill_common::global_settings::validate_instance_banner(v).map_err(|e| { + error::Error::BadRequest(format!("{INSTANCE_BANNER_SETTING}: {e}")) + })?; + } + } + } _ => {} } Ok(()) @@ -1306,12 +1328,21 @@ pub async fn get_global_setting( && key != AUTOMATE_USERNAME_CREATION_SETTING && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING + // `wmill hub pull` reads it from a job, and no job token clears the gate. It binds an + // offline license only together with `license_key`, which stays gated. + && key != UNIQUE_ID_SETTING && key != HUB_ACCESSIBLE_URL_SETTING && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING && key != HTTP_ROUTE_WORKSPACED_ROUTE_SETTING + // The route editor shows the inherited default to whoever is editing a + // trigger, who is usually not a superadmin. Not a secret either: any + // browser discovers the list by reading Access-Control-Allow-Origin off + // a response. + && key != HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING && key != WS_BASE_URL_SETTING + && key != INSTANCE_BANNER_SETTING { require_super_admin(&db, &authed).await?; } diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 47b520736a..260df2f969 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -657,15 +657,17 @@ async fn logout( let t_prefix = token.get(..TOKEN_PREFIX_LEN).unwrap_or(&token); let email = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT { - sqlx::query_scalar!( + // A guest's browser session is a session too: this is its one user-driven revocation. + sqlx::query_scalar::<_, Option>( "WITH email_lookup AS ( SELECT email FROM token WHERE token_hash = $1 ) DELETE FROM token - WHERE email = (SELECT email FROM email_lookup) AND label = 'session' + WHERE email = (SELECT email FROM email_lookup) + AND label IN ('session', 'guest_session') RETURNING email", - t_hash ) + .bind(&t_hash) .fetch_optional(&mut *tx) .await? } else { @@ -745,7 +747,30 @@ async fn whoami( Path(w_id): Path, authed: ApiAuthed, ) -> JsonResult { + let is_guest = windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()); let ApiAuthed { username, email, is_admin, groups, folders, .. } = authed; + // A guest would otherwise fall through to the non-member branch below and be + // handed a `superadmin` role. Answer it here, as the operator-shaped identity it is. + if is_guest { + return Ok(Json(UserInfo { + workspace_id: w_id, + email, + username, + name: None, + is_admin: false, + is_super_admin: false, + created_at: chrono::Utc::now(), + groups: vec![], + operator: true, + disabled: false, + role: Some("guest".to_string()), + folders_read: vec![], + folders: vec![], + folders_owners: vec![], + is_service_account: false, + non_member: true, + })); + } let user = get_user(&w_id, &username, &db).await?; // Only treat the row as "this user is a member" when its email matches; the // derived username is instance-unique so a match on a different email should @@ -2124,6 +2149,28 @@ async fn change_user_email( .execute(&mut *tx) .await?; + // An app draft carries a copy of the deployed policy, principal included. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2"#, + &new_principal, + &old_principal + ) + .execute(&mut *tx) + .await?; + + // A raw-app draft persists the address the client read back too. The deploy sends it beside + // the principal, where an address naming somebody else is rejected — and unlike a live read + // it never refreshes on its own. Same group guard as the deployed policy above, plus the + // `IS NULL` arm: without it the predicate is `NULL` for a draft with no principal, which is + // neither true nor false, so those rows would be skipped. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of_email' = $2 AND (value->'policy'->>'on_behalf_of' IS NULL OR value->'policy'->>'on_behalf_of' NOT LIKE 'g/%')"#, + &new_email, + &old_email + ) + .execute(&mut *tx) + .await?; + // A folder's default rules are an ordered array, first match wins, so the rewrite has to // preserve their order. A rule left on the old address makes `ensure_permissioned_as_exists` // reject the creation of every runnable the rule matches. @@ -2301,9 +2348,9 @@ async fn change_user_email( ) .await?; - // Read back inside the transaction: the address is derived at dispatch through a cache - // that nothing else evicts, so without this a job pushed in the next 60s would resolve - // the old address and with it the wrong superadmin flag and instance groups. + // Read back inside the transaction so this process can evict its own keys immediately. + // `notify_user_email_change` reaches every replica for the same change, but asynchronously, + // and this one is the replica that just served the request. let memberships = sqlx::query_scalar!("SELECT workspace_id FROM usr WHERE email = $1", &new_email) .fetch_all(&mut *tx) @@ -2882,7 +2929,12 @@ pub async fn create_session_token<'c>( .execute(&mut **tx) .await?; - let mut cookie = Cookie::new(COOKIE_NAME, token.clone()); + set_session_cookie(&cookies, &token, *MAX_SESSION_VALIDITY_SECONDS); + Ok(token) +} + +fn set_session_cookie(cookies: &Cookies, token: &str, validity_seconds: i64) { + let mut cookie = Cookie::new(COOKIE_NAME, token.to_string()); cookie.set_secure(IS_SECURE.load(std::sync::atomic::Ordering::Relaxed)); cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax)); cookie.set_http_only(true); @@ -2892,9 +2944,116 @@ pub async fn create_session_token<'c>( } let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc(); - expire += time::Duration::seconds(*MAX_SESSION_VALIDITY_SECONDS); + expire += time::Duration::seconds(validity_seconds); cookie.set_expires(expire); cookies.add(cookie); +} + +lazy_static::lazy_static! { + /// A guest session is the only credential held by someone with no account, so + /// there is nothing to disable when the workspace revokes guest access or the + /// identity provider removes them — the expiry is the revocation. Much shorter + /// than a member session for that reason. + static ref GUEST_SESSION_VALIDITY_SECONDS: i64 = std::env::var("GUEST_SESSION_VALIDITY_SECONDS") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(8 * 60 * 60); +} + +/// Mint a browser session for someone the identity provider authenticated who is a +/// member of no workspace, so they can open one guest-mode app. Writes no `password` +/// and no `usr` row: that absence is what keeps a guest off every seat counter, so +/// nothing here may be "helpfully" upgraded into provisioning. +/// +/// Pinned to `w_id` (`AuthCache` matches on `token.workspace_id`): without the pin an +/// `apps:run:` scope would unlock a same-path app elsewhere. So a guest cannot +/// authenticate on any workspace-less route (`/api/users/*`, `/api/settings/*`); a +/// page that needs one for a guest must become workspace-scoped, not loosen the pin. +/// +/// Refuses unless every gate says yes (`guest_app_admits`, then the allowance in +/// `guest_admission`), so no caller can mint where a guest is not wanted, whatever it +/// believed when it decided to call. All that is left to the caller is the +/// authentication of `email`. +pub async fn create_guest_session_token<'c>( + email: &str, + w_id: &str, + app_path: &str, + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + cookies: Cookies, +) -> Result { + use windmill_common::min_version::MIN_VERSION_SUPPORTS_TOKEN_HASH; + + let token = rd_string(32); + let t_hash = windmill_common::auth::hash_token(&token); + let t_prefix = token.get(..TOKEN_PREFIX_LEN).unwrap_or(&token); + let plaintext: Option<&str> = if MIN_VERSION_SUPPORTS_TOKEN_HASH.met().await { + None + } else { + Some(&token) + }; + let scopes = windmill_api_auth::scopes::guest_session_scopes(app_path)?; + + // No account at all (see `has_any_account`): an account holder is refused a guest + // session, never handed a second, cheaper identity. The same helper the JWT arm uses. + if windmill_common::users::has_any_account(&mut **tx, email).await? { + return Err(Error::NotAuthorized( + "an existing account cannot hold a guest session".to_string(), + )); + } + if !windmill_common::workspaces::guest_app_admits(&mut **tx, w_id, app_path).await? { + return Err(Error::NotAuthorized(format!( + "app {app_path} is not open to guests" + ))); + } + windmill_common::workspaces::guest_admission(&mut **tx, email).await?; + + sqlx::query!( + "INSERT INTO token + (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id) + VALUES ($1, $2, $3, $4, $5, now() + ($6 || ' seconds')::interval, false, $7, $8)", + t_hash, + t_prefix, + plaintext as Option<&str>, + email, + windmill_common::auth::GUEST_SESSION_LABEL, + &GUEST_SESSION_VALIDITY_SECONDS.to_string(), + &scopes, + w_id, + ) + .execute(&mut **tx) + .await?; + + // The only durable record that a guest was here, and the set the allowance is + // counted on; not the audit log, see the migration. Idempotent per email, + // workspace and day. + sqlx::query!( + "INSERT INTO guest_activity (email, workspace_id, day) + VALUES ($1, $2, CURRENT_DATE) + ON CONFLICT (email, workspace_id, day) + DO UPDATE SET last_seen_at = now()", + email, + w_id, + ) + .execute(&mut **tx) + .await?; + + audit_log( + &mut **tx, + &AuditAuthor { + email: email.to_string(), + username: email.to_string(), + username_override: None, + token_prefix: Some(t_prefix.to_string()), + }, + "users.login_guest", + ActionKind::Create, + w_id, + Some(app_path), + Some([("entry", "idp")].into()), + ) + .await?; + + set_session_cookie(&cookies, &token, *GUEST_SESSION_VALIDITY_SECONDS); Ok(token) } @@ -3159,9 +3318,13 @@ async fn update_token_scopes( let mut tx = db.begin().await?; + // A guest-labelled token is never rescoped: its scopes are its whole confinement, + // and after promotion the same email owns an account that could otherwise strip + // them from the still-valid guest credential. Same shape as the relabel guard. let updated: Option = sqlx::query_scalar!( "UPDATE token SET scopes = $1 WHERE email = $2 AND token_prefix = $3 + AND (label IS NULL OR label <> 'guest_session') RETURNING token_prefix", req.scopes.as_deref(), &authed.email, @@ -3172,7 +3335,7 @@ async fn update_token_scopes( let prefix = updated.ok_or_else(|| { Error::NotFound(format!( - "token {token_prefix} not found or not owned by user" + "token {token_prefix} not found, not owned by user, or not rescopable" )) })?; @@ -3242,6 +3405,7 @@ async fn update_token_label( WHERE email = $2 AND token_prefix = $3 AND (label IS NULL OR ( label <> 'session' + AND label <> 'guest_session' AND lower(label) NOT LIKE 'ephemeral%' AND label <> 'debugger-token' AND label NOT LIKE 'mcp-oauth-%' diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 6ee55ed003..07bef364c8 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -19,6 +19,7 @@ use windmill_common::{ db::UserDB, error::JsonResult, jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE}, + queue_metrics::{read_queue_metrics_series, QueueMetricsSeries}, utils::{paginate, Pagination}, worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE}, workspaces::workspace_with_fork_ancestors, @@ -38,6 +39,8 @@ pub fn global_service() -> Router { ) .route("/get_default_tags", get(get_default_tags)) .route("/queue_metrics", get(get_queue_metrics)) + .route("/queue_metrics_series", get(get_queue_metrics_series)) + .route("/queue_status", get(get_queue_status)) .route("/queue_counts", get(get_queue_counts)) .route("/queue_running_counts", get(get_queue_running_counts)) .route( @@ -270,10 +273,16 @@ async fn get_queue_metrics( ) -> JsonResult> { require_devops_role(&db, &authed).await?; + // The API declares every `value` a number, so a climbing delay, stored as its head's wait + // start, is returned as the delay at the time of its sample. let queue_metrics = sqlx::query_as!( QueueMetric, "WITH queue_metrics as ( - SELECT id, value, created_at + SELECT id, created_at, + CASE WHEN jsonb_typeof(value) = 'object' + THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric) + ELSE value + END AS value FROM metrics WHERE id LIKE 'queue_%' AND created_at > now() - interval '14 day' @@ -289,6 +298,88 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } +#[derive(Deserialize)] +struct QueueMetricsSeriesQuery { + window_secs: Option, +} + +const QUEUE_METRICS_DEFAULT_WINDOW_SECS: i64 = 24 * 3600; +/// Retention of queue metrics, past which there is nothing left to read. +const QUEUE_METRICS_MAX_WINDOW_SECS: i64 = 14 * 24 * 3600; + +async fn get_queue_metrics_series( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult { + require_devops_role(&db, &authed).await?; + + let window = query + .window_secs + .unwrap_or(QUEUE_METRICS_DEFAULT_WINDOW_SECS) + .clamp(60, QUEUE_METRICS_MAX_WINDOW_SECS); + Ok(Json(read_queue_metrics_series(&db, window as f64).await?)) +} + +#[derive(Serialize)] +struct QueueTagStatus { + tag: String, + /// Jobs due for more than 3 seconds that no worker has picked up. + waiting: u32, + /// How long the job the next pull would take has been waiting, in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + delay: Option, + running: i64, + /// Workers that pinged in the last minute and pull this tag. + workers: i64, +} + +/// Every tag with jobs waiting or running, read live from the queue. A backlog on a tag no live +/// worker pulls waits for one to start: a worker group scaling up from zero, or none at all for a +/// tag nobody serves. +async fn get_queue_status( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_devops_role(&db, &authed).await?; + + let backlog = windmill_common::queue::get_queue_stats(&db).await?; + let backlog_tags = backlog.keys().cloned().collect::>(); + // A job's tag is resolved before it is queued (per-workspace and dedicated worker tags + // included), and the pull matches it exactly against the worker's tags, so containment is + // exact here too. + let rows = sqlx::query!( + "WITH running AS ( + SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag + ) + SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\", + (SELECT count(*) FROM worker_ping w + WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag] + ) AS \"workers!\" + FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag) + LEFT JOIN running r ON r.tag = t.tag + ORDER BY t.tag", + &backlog_tags[..], + ) + .fetch_all(&db) + .await?; + + Ok(Json( + rows.into_iter() + .map(|row| { + let stat = backlog.get(&row.tag); + QueueTagStatus { + waiting: stat.map_or(0, |s| s.count), + delay: stat.map(|s| s.delay), + running: row.running, + workers: row.workers, + tag: row.tag, + } + }) + .collect(), + )) +} + async fn get_queue_counts( authed: ApiAuthed, Extension(db): Extension, diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index bb4310d1bc..ef5ea37e1d 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -416,6 +416,16 @@ async fn run_datatable_migrations( let applied_versions = read_applied_versions_on_client(&client, &datatable_name).await?; + // How the user scoped the run, for the counter emitted on the first migration + // that lands below. + let scope = if query.only.is_some() { + "only" + } else if query.up_to.is_some() { + "up_to" + } else { + "all" + }; + let mut applied = Vec::new(); for m in migrations { if let Some(only) = query.only { @@ -453,6 +463,14 @@ async fn run_datatable_migrations( )) })?; applied.push(AppliedMigration { version: m.timestamp, name: m.name }); + // One event per run that moved the data table forward, emitted on the + // first migration that lands rather than after the loop: a later one + // failing returns early, and that run still advanced the data table. A + // run with nothing pending stays uncounted — it is the common outcome of + // opening the list and would drown out the runs that did something. + if applied.len() == 1 { + windmill_common::feature_usage::log_feature_usage("datatable", "migration_run", scope); + } } Ok(Json(RunDatatableMigrationsResult { applied })) @@ -594,6 +612,12 @@ async fn rollback_datatable_migrations( )) })?; + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_rollback", + if query.only.is_some() { "only" } else { "last" }, + ); + Ok(Json(RollbackDatatableMigrationsResult { rolled_back: vec![RolledBackMigration { version, name: definition.name }], })) @@ -824,6 +848,8 @@ async fn enable_datatable_migrations( ) .await?; + windmill_common::feature_usage::log_feature_usage("datatable", "migrations_toggled", "on"); + Ok(format!( "Enabled migrations for data table {datatable_name}" )) @@ -892,6 +918,8 @@ async fn disable_datatable_migrations( .await?; } + windmill_common::feature_usage::log_feature_usage("datatable", "migrations_toggled", "off"); + Ok(format!( "Disabled migrations for data table {datatable_name} and deleted its migrations" )) @@ -1134,6 +1162,8 @@ async fn create_datatable_migration( ) .await?; + windmill_common::feature_usage::log_feature_usage("datatable", "migration_created", "manual"); + Ok(Json(DatatableMigration { datatable: datatable_name, timestamp, @@ -1371,6 +1401,20 @@ async fn upsert_datatable_migration( ) .await?; + // An unchanged re-push is not counted: `wmill sync push` sends every migration + // on every sync, so counting those would swamp the definitions people write. + if !unchanged { + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_created", + if existing.is_none() { + "synced" + } else { + "edited" + }, + ); + } + Ok(format!( "Upserted migration {} in {}", payload.timestamp, datatable_name @@ -1477,6 +1521,12 @@ async fn generate_initial_datatable_migration( ) .await?; + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_created", + "initial_snapshot", + ); + Ok(Json(DatatableMigration { datatable: datatable_name, timestamp, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index cb5c639d46..3a2b1d715a 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -12,10 +12,10 @@ use windmill_api_auth::{ }; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; -use windmill_dep_map::lock_hash::record_lock_hashes_for_workspace; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; use windmill_common::webhook::WebhookShared; use windmill_common::{BASE_URL, DB}; +use windmill_dep_map::lock_hash::record_lock_hashes_for_workspace; use axum::{ extract::{Extension, Path, Query}, @@ -151,6 +151,9 @@ pub fn workspaced_service() -> Router { ) .route("/edit_deploy_ui_config", post(edit_deploy_ui_config)) .route("/edit_default_app", post(edit_default_app)) + .route("/edit_guest_access", post(edit_guest_access)) + .route("/edit_guest_jwt_key", post(edit_guest_jwt_key)) + .route("/guest_usage", get(get_guest_usage)) .route("/default_app", get(get_default_app)) .route( "/default_scripts", @@ -317,6 +320,17 @@ pub struct WorkspaceSettings { #[serde(skip_serializing_if = "Option::is_none")] pub public_app_execution_limit_per_minute: Option, pub error_handler_fallback_to_instance_alerts: bool, + /// Whether this workspace admits guest sessions (`ExecutionMode::Guest`). An app's + /// own `execution_mode: guest` is inert while this is off. + pub guest_access_enabled: bool, + /// The key a guest JWT is verified against: a PEM public key, or a JWKS URL, at most + /// one (a DB CHECK enforces it). Public material, not a secret, so it is admin- + /// readable here. `None`/`None` falls back to the instance issuer (`JWT_EXT_JWKS_URL`) + /// off cloud, or accepts no JWT guest if none is set; `guest_access_enabled` is the switch. + #[serde(skip_serializing_if = "Option::is_none")] + pub guest_jwt_public_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub guest_jwt_jwks_url: Option, } /// Subset of `WorkspaceSettings` that is safe to return to any workspace @@ -339,6 +353,9 @@ pub struct WorkspacePublicSettings { pub teams_team_guid: Option, #[serde(skip_serializing_if = "Option::is_none")] pub mute_critical_alerts: Option, + /// Not sensitive, and the app editor needs it to say whether the guest rung is + /// live -- an app can be set to `guest` while the workspace has guests off. + pub guest_access_enabled: bool, #[serde(skip_serializing_if = "Option::is_none")] pub deploy_ui: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1073,7 +1090,10 @@ async fn get_settings( error_handler, success_handler, public_app_execution_limit_per_minute, - error_handler_fallback_to_instance_alerts + error_handler_fallback_to_instance_alerts, + guest_access_enabled, + guest_jwt_public_key, + guest_jwt_jwks_url FROM workspace_settings WHERE @@ -1112,6 +1132,7 @@ async fn get_public_settings( teams_team_name, teams_team_guid, mute_critical_alerts, + guest_access_enabled, deploy_ui, large_file_storage, datatable @@ -1132,6 +1153,18 @@ async fn get_public_settings( Ok(Json(settings)) } +/// The instance's standing against the guest allowance: counts only, no emails, so any +/// member may read it. Instance-wide, since a licence is per instance and one email is +/// one guest however many workspaces it opens; the settings card and the editor's +/// Guests rung show it so nobody discovers the cap from a visitor's complaint. +async fn get_guest_usage( + _authed: ApiAuthed, + Extension(db): Extension, + Path(_w_id): Path, +) -> JsonResult { + Ok(Json(windmill_common::workspaces::guest_usage(&db).await?)) +} + #[derive(Deserialize)] pub struct GitSyncDeployModeQuery { /// The branch the caller would push. @@ -3491,6 +3524,9 @@ async fn edit_datatable_config( // Migrations opt-in is owned by the enable/disable endpoints, not this config // form: preserve each existing data table's flag, and default brand-new data // tables to enabled. + // Counted here rather than after the write because this is where a rename is + // still distinguishable from a creation; emitted once the commit lands. + let mut created_substrates: Vec<&'static str> = Vec::new(); for (name, dt) in new_config.settings.datatables.iter_mut() { let lookup = rename_src .get(name.as_str()) @@ -3498,7 +3534,15 @@ async fn edit_datatable_config( .unwrap_or(name.as_str()); dt.migrations_enabled = match old_datatables.get(lookup) { Some(old) => old.migrations_enabled, - None => Some(true), + None => { + // Keyed by how the substrate is serialized into `workspace_settings`, + // so these line up with the `datatable_configured` adoption counts. + created_substrates.push(match dt.database.resource_type { + DataTableCatalogResourceType::Instance => "instance", + DataTableCatalogResourceType::Postgresql => "postgresql", + }); + Some(true) + } }; } @@ -3556,6 +3600,10 @@ async fn edit_datatable_config( tx.commit().await?; + for substrate in created_substrates { + windmill_common::feature_usage::log_feature_usage("datatable", "created", substrate); + } + crate::datatable_migrations::record_datatable_cascade_deployments( &authed, &db, @@ -3938,6 +3986,7 @@ async fn edit_git_sync_config( clear_client_supplied_auto_pull_state(ap); } repo.open_pr_error = None; + repo.credential = None; } reject_parent_only_git_sync_settings_on_fork( &db, @@ -4038,6 +4087,7 @@ async fn edit_git_sync_config( continue; }; repo.open_pr_error = old.open_pr_error.clone(); + repo.credential = old.credential.clone(); if let (Some(new_ap), Some(old_ap)) = (repo.auto_pull.as_mut(), old.auto_pull.as_ref()) { @@ -4083,6 +4133,7 @@ async fn edit_git_sync_config( .flatten() .and_then(|v| serde_json::from_value(v).ok()); let removed_webhooks: Vec<(String, i64)> = existing + .as_ref() .map(|e| { e.repositories .iter() @@ -4116,6 +4167,19 @@ async fn edit_git_sync_config( // `sync_repo_webhook` writes back the webhook fields it changes itself: // the remote hook and the record of it have to move together, so // persisting them out here would let one land without the other. + // Before the webhook reconcile, which decides whether this repo can have + // one from the credential this records. Also puts a short-lived or + // under-scoped token in front of the operator while they are still on the + // settings page, rather than when it expires. + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); @@ -4169,6 +4233,7 @@ async fn edit_git_sync_repository( clear_client_supplied_auto_pull_state(ap); } new_config.repository.open_pr_error = None; + new_config.repository.credential = None; reject_parent_only_git_sync_settings_on_fork( &db, &w_id, @@ -4293,6 +4358,7 @@ async fn edit_git_sync_repository( // from the UI cannot revert what the poller/webhook layer wrote. let mut updated = new_config.repository; updated.open_pr_error = existing_repo.open_pr_error.clone(); + updated.credential = existing_repo.credential.clone(); match (updated.auto_pull.as_mut(), existing_repo.auto_pull.as_ref()) { (Some(new_ap), Some(old_ap)) => { new_ap.last_synced_sha = old_ap.last_synced_sha.clone(); @@ -4350,6 +4416,15 @@ async fn edit_git_sync_repository( .iter_mut() .find(|r| r.git_repo_resource_path == new_config.git_repo_resource_path) { + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); } @@ -4489,6 +4564,10 @@ async fn delete_git_sync_repository( } } + // The stored credential is deliberately left alone: it belongs to the git + // repository resource, which this endpoint does not delete, and the resource + // still authenticates with it for connection tests and commit lookups. + // Trigger git sync for repository deletion handle_deployment_metadata( &authed.email, @@ -4595,6 +4674,118 @@ async fn edit_default_app( )); } +#[derive(Deserialize)] +struct EditGuestAccess { + guest_access_enabled: bool, +} + +/// Turn guest sessions on or off for this workspace. Off by default, and off is +/// authoritative and immediate: the switch is re-read where a guest session is +/// minted (`guest_app_admits`) and at the auth door on every guest request, so an app +/// whose policy already says `guest` — pushed by git-sync, say — closes to guests on +/// the next request, sessions already issued included. +async fn edit_guest_access( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(EditGuestAccess { guest_access_enabled }): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + if guest_access_enabled { + windmill_common::workspaces::require_guest_support()?; + } + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace_settings SET guest_access_enabled = $1 WHERE workspace_id = $2", + guest_access_enabled, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_guest_access", + ActionKind::Update, + &w_id, + Some(&guest_access_enabled.to_string()), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!( + "Guest access set to {guest_access_enabled} for workspace {w_id}" + )) +} + +#[derive(Deserialize)] +struct EditGuestJwtKey { + /// A PEM public key (RS or ES family), or a JWKS URL, at most one. Both empty clears the + /// workspace key; verification then falls back to the instance issuer (`JWT_EXT_JWKS_URL`) + /// off cloud, or refuses the JWT if none is set. The off-switch is `guest_access_enabled`. + public_key: Option, + jwks_url: Option, +} + +/// Configure the key a guest JWT (`jwt_guest_`) is verified against for this workspace. +/// Workspace-admin gated, like the guest switch: guests are free up to the instance +/// allowance on any plan, so configuring their key needs no licence. The key is +/// validated before it is stored so a typo is refused here, not silently on every guest +/// later: a PEM must parse as an RS/ES public key (HS* has no PEM form and is +/// unreachable), and a JWKS URL must be fetchable and hold at least one usable signing key. +async fn edit_guest_jwt_key( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(EditGuestJwtKey { public_key, jwks_url }): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + let public_key = public_key.filter(|s| !s.trim().is_empty()); + let jwks_url = jwks_url.filter(|s| !s.trim().is_empty()); + if public_key.is_some() && jwks_url.is_some() { + return Err(Error::BadRequest( + "Set a PEM public key or a JWKS URL, not both".to_string(), + )); + } + // Clearing stays allowed wherever guests are: a key nobody can use is still worth + // removing. + if public_key.is_some() || jwks_url.is_some() { + windmill_common::workspaces::require_guest_support()?; + } + if let Some(pem) = public_key.as_deref() { + windmill_common::guest_jwt::decoding_key_from_pem(pem)?; + } + if let Some(url) = jwks_url.as_deref() { + windmill_common::guest_jwt::fetch_jwks(url).await?; + } + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace_settings SET guest_jwt_public_key = $1, guest_jwt_jwks_url = $2 WHERE workspace_id = $3", + public_key, + jwks_url, + &w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "workspaces.edit_guest_jwt_key", + ActionKind::Update, + &w_id, + None, + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Guest JWT key updated for workspace {w_id}")) +} + async fn edit_default_scripts( authed: ApiAuthed, Extension(db): Extension, @@ -5865,7 +6056,7 @@ async fn clone_triggers_and_schedules( path, route_path, route_path_key, script_path, is_flow, workspace_id, edited_by, edited_at, extra_perms, authentication_method, http_method, static_asset_config, is_static_website, workspaced_route, wrap_body, - raw_string, authentication_resource_path, summary, description, + raw_string, allowed_origins, authentication_resource_path, summary, description, error_handler_path, error_handler_args, retry, request_type, mode, permissioned_as, labels ) @@ -5873,7 +6064,7 @@ async fn clone_triggers_and_schedules( path, route_path, route_path_key, script_path, is_flow, $1, edited_by, edited_at, extra_perms, authentication_method, http_method, static_asset_config, is_static_website, workspaced_route, wrap_body, - raw_string, authentication_resource_path, summary, description, + raw_string, allowed_origins, authentication_resource_path, summary, description, error_handler_path, error_handler_args, retry, request_type, 'disabled'::TRIGGER_MODE, permissioned_as, labels FROM http_trigger @@ -6154,9 +6345,10 @@ async fn update_workspace_settings( // Auto-pull and fork PRs are parent-owned and must not be inherited: // the fork would otherwise carry the parent's webhook id (turning off // auto-pull on the fork would delete the parent's webhook). A fork - // still inherits the push-direction config and the installation. - // Repo → fork sync is driven by the parent's webhook/poller - // (`sync_forks`), which routes the fork's `wm-fork/**` branch into it. + // still inherits the push-direction config, the installation, and the + // recorded credential status, which describes the repository rather + // than belonging to either workspace and would otherwise leave the + // fork unqualified for managed features until its first check. r.auto_pull = None; r.fork_open_prs = false; r.open_pr_error = None; @@ -6475,7 +6667,7 @@ async fn clone_scripts( } /// The parsed dbt graph a deployed script carries: its models, their SQL and -/// tests, and the `ref()` lineage between them. +/// tests, and the `ref()` and column-level lineage between them. /// /// Keyed on (workspace_id, script_path, script_hash), and the fork keeps every /// script's hash, so each row moves across as itself. @@ -6493,11 +6685,11 @@ async fn clone_dbt_graph( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at) + columns, column_schema, freshness, raw_code, original_file_path, ingested_at) SELECT $2, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at + columns, column_schema, freshness, raw_code, original_file_path, ingested_at FROM dbt_node WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", source_workspace_id, @@ -6517,6 +6709,24 @@ async fn clone_dbt_graph( ) .execute(&mut **tx) .await?; + // Column lineage travels with the rest of the graph, and it has to: the + // snapshot's digest covers it, so a fork missing these rows recomputes the + // digest the source stored, matches, and stores nothing — leaving the + // lineage gone until someone redeploys, which is the failure this whole + // function exists to prevent. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind, + ingested_at) + SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column, + child_unique_id, child_column, lineage_kind, ingested_at + FROM dbt_column_edge + WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + source_workspace_id, + target_workspace_id + ) + .execute(&mut **tx) + .await?; sqlx::query!( "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest, relation_root_at_last_ingest, ingested_at) @@ -11106,6 +11316,7 @@ async fn load_workspace_authed( token_prefix: base_authed.token_prefix.clone(), read_only: base_authed.read_only, job_id: base_authed.job_id, + credential_expiry: base_authed.credential_expiry, }); }; @@ -11138,6 +11349,7 @@ async fn load_workspace_authed( token_prefix: base_authed.token_prefix.clone(), read_only: base_authed.read_only, job_id: base_authed.job_id, + credential_expiry: base_authed.credential_expiry, }) } diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 1968437e40..e75f0d1ef2 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -113,7 +113,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -187,6 +187,13 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + info!("Updating guest_activity table"); + sqlx::query("UPDATE guest_activity SET workspace_id = $1 WHERE workspace_id = $2") + .bind(&rw.new_id) + .bind(&old_id) + .execute(&mut *tx) + .await?; + info!("Updating workspace_invite table"); sqlx::query!( "UPDATE workspace_invite SET workspace_id = $1 WHERE workspace_id = $2", @@ -1112,6 +1119,13 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + // Unlike the rest of this list, this also moves an instance-wide figure: the guest + // allowance and the seats past it are counted over every workspace's rows. + sqlx::query("DELETE FROM guest_activity WHERE workspace_id = $1") + .bind(&w_id) + .execute(&mut *tx) + .await?; + sqlx::query!("DELETE FROM token WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 436fee424f..250f958251 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.803.0 + version: 1.811.1 title: Windmill API contact: @@ -861,6 +861,34 @@ paths: items: $ref: "#/components/schemas/ExternalJwtToken" + /users/guests: + get: + summary: list the distinct guests of the trailing window (superadmin only) + description: >- + The set the guest allowance is counted on: every distinct email that held a + guest session in the last `window_days`, with the workspaces it opened and the + days it was first and last seen, most recently seen first. `usage` is the + instance's standing against the allowance and the meter. + operationId: listGuests + tags: + - user + parameters: + - name: page + in: query + schema: + type: integer + - name: per_page + in: query + schema: + type: integer + responses: + "200": + description: the guests of the window and the allowance they count against + content: + application/json: + schema: + $ref: "#/components/schemas/GuestList" + /users/onboarding: post: summary: Submit user onboarding data @@ -2713,7 +2741,7 @@ paths: /w/{workspace}/github_app/token: post: - summary: get github app token + summary: get the git credential for a git-sync job (GitHub App token, or the credential stored for the repository) operationId: getGithubAppToken tags: - workspace @@ -2733,7 +2761,7 @@ paths: - job_token responses: "200": - description: github app token + description: git credential content: application/json: schema: @@ -2877,6 +2905,131 @@ paths: "200": description: Successfully imported the installation + /w/{workspace}/git_sync/gitlab/projects: + post: + tags: + - Git Sync + summary: List the GitLab projects a token can sync + description: >- + Lists the projects the supplied GitLab token can push to, so a git + repository resource can be filled in without hand-writing a project + path. The token is used for this call only and is never stored. + Requires workspace admin. + operationId: listGitlabProjects + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + base_url: + type: string + description: The GitLab instance, e.g. https://gitlab.com + token: + type: string + description: A project access token with the api scope, or a group token that reaches the project + search: + type: string + description: Narrow the list to projects matching this text + required: + - base_url + - token + responses: + "200": + description: the projects the token can sync + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/GitlabProject" + + /w/{workspace}/git_sync/credential/origin: + get: + tags: + - Git Sync + summary: Where a repository's credential comes from + description: >- + Whether Windmill holds this repository's access token, and which host it + talks to. `held` means this workspace stores it, `borrowed` means an + ancestor does and it is not this workspace's to replace. Both absent + means the repository authenticates with whatever its URL carries. + Returns no secret. Requires workspace admin. + operationId: getCredentialOrigin + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: >- + Path of the git repository resource, with or without the `$res:` + prefix. A path rather than a URL, because a resource URL may carry a + token and a URL in a query string lands in logs. + schema: + type: string + responses: + "200": + description: where the credential comes from + content: + application/json: + schema: + type: object + properties: + origin: + type: string + enum: + - held + - borrowed + provider: + type: string + enum: + - gitlab + + /w/{workspace}/git_sync/credential: + post: + tags: + - Git Sync + summary: Store the credential for a git repository + description: >- + Stores the access token a git repository authenticates with, so it does + not have to be written into the repository URL or a workspace variable. + The token is write-only: it is served only to a git-sync job that + presents its own job token, and a fork of this workspace reads this + copy instead of holding one of its own. Requires workspace admin. + operationId: setGitCredential + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + repo_url: + type: string + description: >- + The repository the credential is for, and the key it is + stored under. It is served for this repository and no other, + so repointing a resource elsewhere cannot carry the token + along. + token: + type: string + description: The access token, as pasted + required: + - repo_url + - token + responses: + "200": + description: the credential was stored + content: + text/plain: + schema: + type: string + /w/{workspace}/github_app/ghes_installation_callback: post: summary: GHES installation callback @@ -3667,8 +3820,12 @@ paths: $ref: "#/components/schemas/WorkspaceDeployUISettings" mute_critical_alerts: type: boolean + guest_access_enabled: + type: boolean + description: Whether this workspace admits guest sessions. An app's own `guest` execution mode is inert while this is false. required: - workspace_id + - guest_access_enabled /w/{workspace}/workspaces/get_settings: get: @@ -3750,6 +3907,15 @@ paths: error_handler_fallback_to_instance_alerts: type: boolean description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. + guest_access_enabled: + type: boolean + description: Whether this workspace admits guest sessions. An app's own `guest` execution mode is inert while this is false. + guest_jwt_public_key: + type: string + description: PEM public key a guest JWT (`jwt_guest_`) is verified against for this workspace. Mutually exclusive with `guest_jwt_jwks_url`. + guest_jwt_jwks_url: + type: string + description: JWKS URL a guest JWT (`jwt_guest_`) is verified against for this workspace. Mutually exclusive with `guest_jwt_public_key`. /w/{workspace}/workspaces/get_deploy_to: get: @@ -5739,6 +5905,99 @@ paths: schema: type: string + /w/{workspace}/workspaces/edit_guest_access: + post: + summary: enable or disable guest sessions for this workspace + description: >- + Guests are people the identity provider authenticates who have no Windmill + account; the `guest` app execution mode admits them. Off by default. Re-read + where a guest session is minted and at the auth door on every guest request, so + turning it off takes effect immediately, for sessions already issued and for + apps whose policy already says `guest`. Turning it *on* is refused with a 400 + where guests are unavailable (the shared cloud); turning it off always works. + operationId: editGuestAccess + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Whether guest sessions are admitted + required: true + content: + application/json: + schema: + type: object + properties: + guest_access_enabled: + type: boolean + required: + - guest_access_enabled + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/edit_guest_jwt_key: + post: + summary: set the key guest JWTs are verified against for this workspace + description: >- + A guest JWT (`jwt_guest_`) is minted by the embedding customer's own backend and + verified against this key: a PEM public key (RS/ES family, HS* refused) or a JWKS + URL, at most one. Both empty clears the workspace key; off cloud, verification then + falls back to the instance issuer (`JWT_EXT_JWKS_URL`) if one is set, else no guest + JWT is accepted (`guest_access_enabled` is the on/off switch). Workspace-admin gated. + The key is validated before it is stored. Setting a key is refused with a 400 where + guests are unavailable (the shared cloud); clearing one always works. + operationId: editGuestJwtKey + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: The guest JWT verification key + required: true + content: + application/json: + schema: + type: object + properties: + public_key: + type: string + description: A PEM public key (RS or ES family). + jwks_url: + type: string + description: A JWKS URL whose keys are fetched and refreshed. + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/guest_usage: + get: + summary: the instance's standing against the guest allowance + description: >- + Instance-wide, since a licence is per instance and one email is one guest however + many workspaces it opens. Read by workspace admins and app publishers to see how + close the cap (Community and Pro) or the meter (Enterprise) is. + operationId: getGuestUsage + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: guest usage + content: + application/json: + schema: + $ref: "#/components/schemas/GuestUsage" + /w/{workspace}/workspaces/default_scripts: post: summary: edit default scripts for workspace @@ -8396,6 +8655,84 @@ paths: items: type: string + /w/{workspace}/resources/type/resource_counts: + get: + summary: count the workspace's resources per resource type + operationId: listResourceCountsByType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: resource count per resource_type + content: + application/json: + schema: + type: array + items: + type: object + properties: + resource_type: + type: string + count: + type: integer + required: + - resource_type + - count + + /w/{workspace}/resources/type/hub/info: + get: + summary: list what the hub knows about its resource types + operationId: listHubResourceTypeInfo + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: each hub resource type with its integration and pick count, empty if the hub answers neither read + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + app: + description: the integration the resource type belongs to, which is not always its own name + type: string + picks: + type: integer + required: + - name + - app + - picks + + /w/{workspace}/resources/type/hub/pick/{name}: + post: + summary: record a hub resource type pick + operationId: pickHubResourceType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Name" + responses: + "200": + description: whether the hub recorded the pick + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: + - success + /w/{workspace}/npm_proxy/config: get: summary: get npm proxy configuration @@ -8645,6 +8982,9 @@ paths: properties: name: type: string + picks: + description: how often the integration has been picked, absent on a hub that does not count picks + type: integer required: - name @@ -8806,6 +9146,28 @@ paths: required: - app + /apps_u/guest_entry_by_custom_path/{custom_path}: + get: + summary: whether the app behind a custom path admits guests + description: >- + The custom-path counterpart of `getGuestEntry`. Unauthenticated; 404 unless + the app's execution mode is `guest` AND its workspace has + `guest_access_enabled` AND the instance has not set `guest_access_disabled`, + and never on a deployment where guests are unavailable (the shared cloud). + Returns the workspace too, since a custom URL may not carry it. + operationId: getGuestEntryByCustomPath + tags: + - app + parameters: + - $ref: "#/components/parameters/CustomPath" + responses: + "200": + description: the app is open to guests + content: + application/json: + schema: + $ref: "#/components/schemas/GuestEntry" + /apps_u/public_app_by_custom_path/{custom_path}: get: summary: get public app by custom path @@ -13018,6 +13380,31 @@ paths: schema: type: string + /w/{workspace}/apps_u/guest_entry/{path}: + get: + summary: whether the app behind a share secret admits guests + description: >- + Unauthenticated: what a signed-out visitor reads to learn that signing in + would let them in. 404 unless the app's execution mode is `guest` AND the + workspace has `guest_access_enabled` AND the instance has not set the + `guest_access_disabled` global setting, and never on a deployment where guests + are unavailable (the shared cloud), so it says nothing about apps that are not + open to guests. Discloses only the app path, to a caller already + holding the share secret. + operationId: getGuestEntry + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: the app is open to guests + content: + application/json: + schema: + $ref: "#/components/schemas/GuestEntry" + /w/{workspace}/apps_u/public_app/{path}: get: summary: get public app by secret @@ -16516,6 +16903,7 @@ paths: - can_approve - user_auth_required - approvers + - skin properties: flow_id: type: string @@ -16548,6 +16936,16 @@ paths: hide_cancel: type: boolean description: whether to hide the cancel button in the UI + skin: + type: string + enum: [detailed, minimal] + description: how the approval page presents the request + step_summary: + type: string + description: summary of the approval step, for the page title + flow_summary: + type: string + description: summary of the flow or workflow the approval belongs to approvers: type: array items: @@ -18621,6 +19019,49 @@ paths: schema: type: string + /w/{workspace}/native_triggers/{service_name}/setenabled/{external_id}: + post: + summary: set enabled state of native trigger + description: | + Enables or disables a native trigger. A disabled trigger stays registered on the + external service but starts no job when it fires. + Requires write access to the script or flow that the trigger is associated with. + operationId: setNativeTriggerEnabled + tags: + - native_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: service_name + in: path + required: true + schema: + $ref: "#/components/schemas/NativeServiceName" + - name: external_id + in: path + required: true + description: The external ID of the trigger from the external service + schema: + type: string + requestBody: + description: updated enabled state + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + "200": + description: native trigger enabled state updated + content: + text/plain: + schema: + type: string + /w/{workspace}/native_triggers/{service_name}/list: get: summary: list native triggers @@ -21456,6 +21897,98 @@ paths: - id - values + /workers/queue_metrics_series: + get: + summary: get the queue metrics of a time window, as a bounded line per tag + operationId: getQueueMetricsSeries + tags: + - worker + parameters: + - name: window_secs + in: query + required: false + description: how far back to read, in seconds (defaults to one day, capped at the 14-day retention) + schema: + type: integer + responses: + "200": + description: jobs waiting and queue delay per tag, as the vertices of lines joined by straight segments + content: + application/json: + schema: + type: object + properties: + from: + type: integer + description: start of the window, in epoch milliseconds + to: + type: integer + description: end of the window, in epoch milliseconds + tags: + type: array + items: + type: object + properties: + tag: + type: string + count: + type: array + description: "[epoch ms, jobs waiting more than 3 seconds] vertices" + items: + type: array + items: + type: number + delay: + type: array + description: "[epoch ms, seconds the next job has waited] vertices" + items: + type: array + items: + type: number + required: + - tag + - count + - delay + required: + - from + - to + - tags + + /workers/queue_status: + get: + summary: get the live queue status of every tag with jobs waiting or running + operationId: getQueueStatus + tags: + - worker + responses: + "200": + description: queue status per tag + content: + application/json: + schema: + type: array + items: + type: object + properties: + tag: + type: string + waiting: + type: integer + description: jobs due for more than 3 seconds that no worker has picked up + delay: + type: number + description: seconds the job the next pull would take has been waiting, absent when none is + running: + type: integer + workers: + type: integer + description: workers that pinged in the last minute and pull this tag + required: + - tag + - waiting + - running + - workers + /workers/queue_counts: get: summary: get counts of jobs waiting for an executor per tag @@ -24239,6 +24772,72 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/assets/column_lineage: + get: + summary: Column-level lineage of a set of dbt relations + description: > + The direct (`copy` / `mod`) column-to-column lineage the given relations' + columns sit in — the connected component around them, from the engine's + static analysis. Not their own edges, which would stop one hop out since + a column trace walks transitively, and not a whole project's, which + carries model families the selection cannot reach. + + Several relations, answered as one union, because one selection reaches + several: a script's output column can derive from columns of several dbt + models. Unpinned, the component crosses projects — a relation one project + produces is another's source — and the caller's access is decided again + for every project it reaches, so a trace ends where their grants do. A + pinned answer, by version here or by job on the run route, is one + project's. + + Its own endpoint rather than a field on the asset graph: the graph is + folder-wide and polled by a run page, while this is rendered for one + selection at a time. Empty for projects that did not opt into the + analysis pass (`column_lineage: true`), which is the ordinary case. The + indirect `scan` kind is stored but never served: it reaches every output + column of its model. + operationId: getDbtColumnLineage + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. + schema: + type: array + maxItems: 1000 + minItems: 1 + items: + type: string + - name: dbt_script_hash + in: query + description: > + The deployed version a view is drawing, when it is drawing one — the + dbt editor, which shows a single project as of a single deploy. A + version-pinned answer is that version's project alone, the same as a + job-pinned one, and only the unpinned answer crosses projects: a pin + says which stored graph is on screen, and another project's live + graph is not part of it. + + A run's or an editor buffer's own graph is not reachable here: that + pins to a job, and costs the job-read gate — see + `jobs/dbt_column_lineage/{id}`. + schema: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/assets/macros: get: summary: List every workspace DuckDB macro (deployed `// macros` libraries) @@ -24430,6 +25029,53 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/jobs/dbt_column_lineage/{id}: + get: + summary: Get relations' project column lineage as one run saw it + description: > + The same answer as `assets/column_lineage`, for the project version a + single job ran — including the dbt editor's parse of its own buffer, + whose graph belongs to that job and is reachable no other way. One + project answers here, the one the run is of, since the graph this + annotates is that project's too. Authorized through the job, the same + gate as `dbt_graph`. Reaching the run is not on its own enough to read + the project: a caller with no access to the script gets its relations and + `ref()` edges from `dbt_graph` and an empty answer here, exactly as that + endpoint redacts the model's SQL. + operationId: getDbtRunColumnLineage + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: The job whose graph the lineage is read from + schema: + type: string + format: uuid + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. + schema: + type: array + maxItems: 1000 + minItems: 1 + items: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/jobs/run_progress/{id}: get: summary: List the per-relation progress one job has recorded so far @@ -25306,6 +25952,30 @@ paths: schema: type: string + /w/{workspace}/hub/projects: + get: + summary: list the hub's published projects + description: | + Forwards to the configured Hub's public project catalogue and returns its + status code and raw response body. Readable by any workspace member: the + listing is not workspace-scoped, and it is proxied only because the Hub's + listing endpoint sends no CORS header. Refused with 400 when the instance + has the Hub disabled, in which case no outbound request is made. + operationId: listHubProjects + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + "400": + description: the Hub is disabled on this instance + /w/{workspace}/hub/project: get: summary: get the hub project linked to a workspace folder @@ -26023,6 +26693,40 @@ components: drawn identically and the ambiguity would otherwise just move into the editor. Omitted for the unpinned workspace graph, which spans every project and so has no one time. + DbtColumnLineage: + type: object + description: >- + The direct column-to-column lineage the asked-for relations' columns sit + in — the connected component around them — in the terms the canvas draws: + relations and columns, never dbt's node ids. + required: [edges, truncated] + properties: + edges: + type: array + items: + type: object + required: [from_asset_path, from_column, to_asset_path, to_column, kind] + properties: + from_asset_path: + type: string + from_column: + type: string + to_asset_path: + type: string + to_column: + type: string + kind: + type: string + description: >- + dbt's own word for how the value travelled — `copy` + (passthrough) or `mod` (transformed). Not an enum: the engine + treats the set as open. + truncated: + type: boolean + description: >- + The component reaches further than `edges`, which holds the part + nearest the asked-for relations. A trace that stops short is + otherwise indistinguishable from one that ends. DbtAssetProvenance: type: object description: >- @@ -26070,7 +26774,23 @@ components: columns: type: object additionalProperties: true - description: Declared column metadata (name -> description). NOT column lineage — `manifest.json` carries none. + description: Declared column metadata (name -> description) — what `manifest.json` carries, which is only the columns an author wrote down. Omitted when the caller cannot read the script. + column_schema: + type: array + description: >- + Every column of the relation, typed and in the order the model + produces them, from the engine's static analysis. Present only for a + project that opted into it, and gated like `columns` and the model's + SQL: a full column list is the shape of what the author wrote. + items: + type: object + required: [name] + properties: + name: + type: string + type: + type: string + description: The declared type where `schema.yml` gives one, else the inferred one. Omitted when neither is known. freshness: type: object additionalProperties: true @@ -28795,6 +29515,82 @@ components: - is_operator - last_used_at + GuestUsage: + type: object + description: >- + Guests are free up to `free_allowance` distinct emails over the trailing + `window_days`. Past that an Enterprise plan meters them (`metered`, four guests + to one seat: `billable_guests`, `guest_seats`); every other plan and build + admits no new email until the count drops. `instance_enabled` is the superadmin + switch (`guest_access_disabled` global setting) every workspace switch sits under. + `available` is whether this deployment can have guests at all: false on the shared + cloud, where guest access requires a self-hosted or dedicated deployment, and every + other field and switch is then moot. + properties: + available: + type: boolean + instance_enabled: + type: boolean + guest_count: + type: integer + format: int64 + window_days: + type: integer + free_allowance: + type: integer + format: int64 + metered: + type: boolean + billable_guests: + type: integer + format: int64 + guest_seats: + type: integer + format: int64 + required: + - available + - instance_enabled + - guest_count + - window_days + - free_allowance + - metered + - billable_guests + - guest_seats + + GuestActivity: + type: object + properties: + email: + type: string + workspaces: + type: array + items: + type: string + first_seen: + type: string + format: date + last_seen: + type: string + format: date + required: + - email + - workspaces + - first_seen + - last_seen + + GuestList: + type: object + properties: + usage: + $ref: "#/components/schemas/GuestUsage" + guests: + type: array + items: + $ref: "#/components/schemas/GuestActivity" + required: + - usage + - guests + NewToken: type: object properties: @@ -29062,6 +29858,8 @@ components: - "igroup.delete" - "igroup.adduser" - "igroup.removeuser" + - "instance_groups.jit_adduser" + - "instance_groups.jit_removeuser" - "variables.decrypt_secret" - "workspaces.read_encryption_key" - "workspaces.edit_command_script" @@ -30269,6 +31067,14 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare @@ -30360,6 +31166,14 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare @@ -30458,6 +31272,14 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare @@ -33203,18 +34025,23 @@ components: type: string execution_mode: type: string - enum: [viewer, publisher, anonymous] + enum: [viewer, publisher, guest, anonymous] description: >- - Who the app's runnables execute as. Optional, and what omitting it - means depends on the operation: creating an app defaults it to - `publisher` (runs on behalf of the app's publisher and requires an - authenticated viewer), while updating one keeps the mode the app is - already deployed under. Either way `anonymous`, which makes the app - publicly executable, is never assumed + Who may open the app, and who its runnables execute as. Optional, and + what omitting it means depends on the operation: creating an app + defaults it to `publisher` (runs on behalf of the app's publisher and + requires an authenticated viewer), while updating one keeps the mode + the app is already deployed under. Neither `anonymous`, which makes + the app publicly executable, nor `guest`, which opens it to anyone the + identity provider authenticates, is ever assumed. A guest is only + admitted where the workspace also has `guest_access_enabled`, which is + checked when the session is minted and again on every guest request on_behalf_of: type: string + description: The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity. on_behalf_of_email: type: string + description: Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected. sandbox: type: boolean description: > @@ -33261,7 +34088,7 @@ components: format: date-time execution_mode: type: string - enum: [viewer, publisher, anonymous] + enum: [viewer, publisher, guest, anonymous] raw_app: type: boolean labels: @@ -34063,9 +34890,58 @@ components: open_pr_error: type: string description: server-owned, last failure opening a PR for a deploy branch of this repo + credential: + $ref: "#/components/schemas/GitCredentialStatus" required: - git_repo_resource_path + GitCredentialStatus: + type: object + description: server-owned, what the repo's own credential reports about itself + properties: + provider: + type: string + enum: + - gitlab + token_id: + type: integer + format: int64 + expires_at: + type: string + format: date + description: absent for a non-expiring token + scopes: + type: array + items: + type: string + rotatable: + type: boolean + description: whether this workspace renews the credential itself + checked_at: + type: integer + format: int64 + error: + type: string + required: + - provider + - rotatable + - checked_at + + GitlabProject: + type: object + description: a GitLab project a token can sync, as the resource form needs it + properties: + path_with_namespace: + type: string + description: nested group path plus project name, which is also GitLab's project id + http_url_to_repo: + type: string + default_branch: + type: string + required: + - path_with_namespace + - http_url_to_repo + AutoPullMode: type: string enum: @@ -35082,6 +35958,17 @@ components: description: Configuration of protection restrictions items: $ref: "#/components/schemas/ProtectionRuleKind" + GuestEntry: + type: object + description: What a signed-out visitor needs to start a guest sign-in. + properties: + workspace_id: + type: string + app_path: + type: string + required: + - workspace_id + - app_path ProtectionRuleKind: type: string enum: @@ -35090,6 +35977,7 @@ components: - RestrictDeployToDeployers - RestrictAnonymousAppDeployment - RestrictPublicRunSharing + - RestrictGuestAppDeployment RuleBypasserGroups: type: array description: Groups that can bypass this ruleset @@ -35241,6 +36129,9 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: Whether the trigger starts a job when it fires required: - external_id - workspace_id @@ -35248,6 +36139,7 @@ components: - script_path - is_flow - service_config + - enabled NativeTriggerWithExternal: type: object @@ -35279,6 +36171,9 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: Whether the trigger starts a job when it fires external_data: type: object nullable: true @@ -35299,6 +36194,7 @@ components: - script_path - is_flow - service_config + - enabled - external_data WorkspaceIntegrations: @@ -35384,6 +36280,12 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: >- + Whether the trigger starts a job when it fires. Honoured on create only, so a + trigger can be registered already paused; an update ignores it and setenabled is + the only way to change an existing trigger's state. Defaults to true. required: - script_path - is_flow diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e5b5f019eb..717be4ad0f 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -17,6 +17,7 @@ use std::str::FromStr; use uuid::Uuid; use windmill_common::cache; use windmill_common::error::Error; +use windmill_common::flows::{ApprovalSkin, Suspend}; use windmill_common::jobs::JobKind; use windmill_common::scripts::ScriptHash; @@ -94,6 +95,17 @@ pub struct ApprovalFormDetails { pub message_str: String, pub urls: ResumeUrls, pub schema: Option, + pub skin: ApprovalSkin, +} + +/// The suspended step an approval message is about, and the flow run it belongs to. +struct ApprovalStep { + created_by: String, + created_at: chrono::NaiveDateTime, + script_path: Option, + parent_job_id: Option, + args: Option>>, + suspend: Option, } #[allow(dead_code)] @@ -205,6 +217,90 @@ pub async fn get_approval_form_details( tracing::debug!("Job ID: {:?}", job_id); + let ApprovalStep { created_by, created_at, script_path, parent_job_id, args, suspend } = + fetch_approval_step(&db, w_id, job_id, flow_step_id).await?; + + let schema = suspend.as_ref().map(|suspend| ResumeFormRow { + resume_form: suspend.resume_form.clone(), + hide_cancel: suspend.hide_cancel, + }); + let skin = suspend.and_then(|s| s.skin).unwrap_or_default(); + + let bold_format = match format { + MessageFormat::Slack => "*{}*", + MessageFormat::Teams => "**{}**", + }; + + let message_str = match skin { + ApprovalSkin::Detailed => { + let args_str = args.map_or("None".to_string(), |a| { + serde_json::from_str::(a.get()) + .ok() + .and_then(|v| serde_json::to_string_pretty(&v).ok()) + .unwrap_or_else(|| a.get().to_string()) + }); + let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); + let script_path_str = script_path.as_deref().unwrap_or("None"); + + let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); + + let mut message_str = format!( + "A workflow has been suspended and is waiting for approval:\n\n\ + {}: {created_by}\n\n\ + {}: {created_at_formatted}\n\n\ + {}: {script_path_str}\n\n\ + {}:\n```\n{args_str}\n```\n\n\ + {}: {parent_job_id_str}\n\n", + bold_format.replace("{}", "Created by"), + bold_format.replace("{}", "Created at"), + bold_format.replace("{}", "Script path"), + bold_format.replace("{}", "Args"), + bold_format.replace("{}", "Flow ID") + ); + + // Append custom message if provided + if let Some(msg) = message { + message_str.push_str(msg); + } + message_str + } + ApprovalSkin::Minimal => format!( + "{}\n\n{}: {created_by}", + message.unwrap_or("Your approval is requested."), + bold_format.replace("{}", "Requested by"), + ), + }; + + tracing::debug!("Schema: {:#?}", schema); + + Ok(ApprovalFormDetails { message_str, urls, schema, skin }) +} + +/// The skin of the approval step `flow_step_id` of the flow running `job_id`. Falls back to +/// the detailed skin when the step cannot be resolved, so a message is still sent. +/// Reads through the unrestricted pool without an authorization check of its own: only the +/// skin, which is not sensitive, leaves this function. +pub(crate) async fn get_approval_step_skin( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: &str, +) -> ApprovalSkin { + match fetch_approval_step(db, w_id, job_id, Some(flow_step_id)).await { + Ok(step) => step.suspend.and_then(|s| s.skin).unwrap_or_default(), + Err(e) => { + tracing::warn!("Could not resolve approval step {flow_step_id} of job {job_id}: {e}"); + ApprovalSkin::default() + } + } +} + +async fn fetch_approval_step( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: Option<&str>, +) -> Result { // TODO: do we have a helper function for this? let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( "WITH job_info AS ( @@ -240,17 +336,17 @@ pub async fn get_approval_form_details( job_id, &w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await .map_err(|e| Error::BadRequest(e.to_string()))? .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string())) .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?; - let flow_data = match cache::job::fetch_flow(&db, &job_kind, script_hash).await { + let flow_data = match cache::job::fetch_flow(db, &job_kind, script_hash).await { Ok(data) => data, Err(_) => { if let Some(parent_job_id) = parent_job_id.as_ref() { - cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await? + cache::job::fetch_preview_flow(db, parent_job_id, raw_flow).await? } else { return Err(Error::BadRequest( "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(), @@ -265,49 +361,12 @@ pub async fn get_approval_form_details( tracing::debug!("Module: {:#?}", module); - let schema = module.and_then(|module| { - module.suspend.as_ref().map(|suspend| ResumeFormRow { - resume_form: suspend.resume_form.clone(), - hide_cancel: suspend.hide_cancel, - }) - }); - - let args_str = args.map_or("None".to_string(), |a| { - serde_json::from_str::(a.get()) - .ok() - .and_then(|v| serde_json::to_string_pretty(&v).ok()) - .unwrap_or_else(|| a.get().to_string()) - }); - let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); - let script_path_str = script_path.as_deref().unwrap_or("None"); - - let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); - - let bold_format = match format { - MessageFormat::Slack => "*{}*", - MessageFormat::Teams => "**{}**", - }; - - let mut message_str = format!( - "A workflow has been suspended and is waiting for approval:\n\n\ - {}: {created_by}\n\n\ - {}: {created_at_formatted}\n\n\ - {}: {script_path_str}\n\n\ - {}:\n```\n{args_str}\n```\n\n\ - {}: {parent_job_id_str}\n\n", - bold_format.replace("{}", "Created by"), - bold_format.replace("{}", "Created at"), - bold_format.replace("{}", "Script path"), - bold_format.replace("{}", "Args"), - bold_format.replace("{}", "Flow ID") - ); - - // Append custom message if provided - if let Some(msg) = message { - message_str.push_str(msg); - } - - tracing::debug!("Schema: {:#?}", schema); - - Ok(ApprovalFormDetails { message_str, urls, schema }) + Ok(ApprovalStep { + created_by, + created_at, + script_path, + parent_job_id, + args, + suspend: module.and_then(|m| m.suspend.clone()), + }) } diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index bf6cbcbc14..fe35e5b989 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -170,6 +170,7 @@ pub fn unauthed_service() -> Router { ) .route("/load_csv_preview/{*path}", get(app_load_csv_preview)) .route("/public_app/{secret}", get(get_public_app_by_secret)) + .route("/guest_entry/{secret}", get(get_guest_entry)) .route("/embed_token/{secret}", get(get_app_embed_token)) .route("/public_resource/{*path}", get(get_public_resource)) .route("/get_data/v/{*id}", get(get_raw_app_data)) @@ -294,6 +295,15 @@ pub type AllowUserResources = Vec; #[serde(rename_all = "lowercase")] pub enum ExecutionMode { Anonymous, + /// Login required, workspace membership not: anyone the instance's identity + /// provider authenticates may open the app, and the runnables execute as the + /// publisher exactly as in [`ExecutionMode::Publisher`]. Such a viewer holds a + /// guest session: an identity with no account at all (no `password` row, no `usr` + /// row anywhere), which is what keeps it off every row-based seat counter; what a + /// guest costs instead is the allowance in `windmill_common::workspaces`. Honored + /// only where `workspace_settings.guest_access_enabled` is on, re-read at the auth + /// door on every guest request. + Guest, /// Default for a policy that omits `execution_mode`. It MUST stay a mode /// that requires an authenticated viewer: an omitted field must never be /// able to publish an app anonymously (publicly executable). @@ -302,6 +312,159 @@ pub enum ExecutionMode { Viewer, } +impl ExecutionMode { + /// The serialized form, matching this enum's `rename_all = "lowercase"`. + pub fn as_str(&self) -> &'static str { + match self { + ExecutionMode::Anonymous => "anonymous", + ExecutionMode::Guest => "guest", + ExecutionMode::Publisher => "publisher", + ExecutionMode::Viewer => "viewer", + } + } +} + +/// The protection rule gating a *transition into* `mode`, if any. Anonymous and +/// guest each widen who may open an app past the workspace's own members, so each +/// carries its own rule; the two member-only modes are ungated. +fn deployment_rule_for_mode(mode: ExecutionMode) -> Option { + match mode { + ExecutionMode::Anonymous => Some(ProtectionRuleKind::RestrictAnonymousAppDeployment), + ExecutionMode::Guest => Some(ProtectionRuleKind::RestrictGuestAppDeployment), + ExecutionMode::Publisher | ExecutionMode::Viewer => None, + } +} + +/// A guest session is scoped to its app by path, so an app whose path the scope +/// grammar cannot hold as one literal (`is_scope_literal_path`) can never admit a +/// guest; refuse the mode at deploy time rather than advertise an app nobody enters. +/// `path` is where the app ends up: on a rename, the destination. +fn refuse_unscopable_guest_app(path: &str, mode: ExecutionMode) -> Result<()> { + if matches!(mode, ExecutionMode::Guest) && !windmill_common::auth::is_scope_literal_path(path) { + return Err(Error::BadRequest(format!( + "app {path} cannot be set to Guests: a path with `:`, `,` or `*`, or a leading `/`, \ + cannot be scoped" + ))); + } + Ok(()) +} + +/// Refuse *widening* an app into guests where the deployment has none +/// (`instance_supports_guests`). Only the transition is refused, like the protection +/// rule below it: an app already stored in the mode — deployed before the instance +/// became a cloud one, or pushed by git-sync — keeps deploying, and keeps being inert, +/// since every guest gate refuses it anyway. `deployed_mode` is what the app is stored +/// as, `None` when it is being created. +fn refuse_guest_mode_where_unavailable( + path: &str, + mode: ExecutionMode, + deployed_mode: Option, +) -> Result<()> { + if !matches!(mode, ExecutionMode::Guest) + || deployed_mode == Some(ExecutionMode::Guest) + || windmill_common::workspaces::instance_supports_guests() + { + return Ok(()); + } + Err(Error::BadRequest(format!( + "app {path} cannot be set to Guests: {}", + windmill_common::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))) +} + +/// Gate a viewer on the app's `execution_mode`, as far as can be decided without an +/// ACL probe. `Ok(true)` means already authorized — anonymous admits anyone, guest +/// admits anyone signed in; `Ok(false)` means the caller is a member and still owes +/// the read-access check its caller performs. +/// +/// A guest is authorized by its token's scope and never by an ACL probe: it holds no +/// `usr` row, so RLS finds nothing for it and every guest would read as having no +/// access. That scope is also what keeps a guest session to the one app it was minted +/// for, even though the mode itself admits anyone signed in. The workspace's guest +/// switch is not checked here: `AuthCache` enforces it for every guest request. +pub fn authorize_non_member_viewer( + mode: ExecutionMode, + app_path: &str, + opt_authed: &Option, +) -> Result { + if matches!(mode, ExecutionMode::Anonymous) { + return Ok(true); + } + let Some(authed) = opt_authed.as_ref() else { + return Err(Error::NotAuthorized( + "App visibility does not allow public access and you are not logged in".to_string(), + )); + }; + let is_guest = windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()); + if matches!(mode, ExecutionMode::Guest) { + if is_guest { + check_scopes(authed, || format!("apps:read:{}", app_path))?; + } + return Ok(true); + } + if is_guest { + return Err(Error::PermissionDenied(format!( + "app {app_path} is not open to guests" + ))); + } + Ok(false) +} + +/// Confines a guest to its app once the app's mode is known; a no-op for every other +/// caller. An anonymous app is open to anyone, so the guest stays the caller there, +/// as itself: the run and the reads that follow it (job results, S3 provenance) must +/// carry one identity. Anywhere else a mismatch is refused. +fn guest_caller_for_mode( + opt_authed: Option, + mode: ExecutionMode, + app_path: &str, +) -> Result> { + let Some(authed) = opt_authed.as_ref() else { + return Ok(None); + }; + if !windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) + || matches!(mode, ExecutionMode::Anonymous) + { + return Ok(opt_authed); + } + check_scopes(authed, || format!("apps:run:{app_path}")) + .or_else(|_| check_scopes(authed, || format!("apps:read:{app_path}")))?; + Ok(opt_authed) +} + +/// [`authorize_non_member_viewer`] plus the member read-access probe, for the +/// entry points that address an app by id. +async fn authorize_app_viewer( + mode: ExecutionMode, + app_path: &str, + app_id: i64, + w_id: &str, + user_db: &UserDB, + opt_authed: &Option, +) -> Result<()> { + if authorize_non_member_viewer(mode, app_path, opt_authed)? { + return Ok(()); + } + let authed = opt_authed + .as_ref() + .ok_or_else(|| Error::internal_err("authorize_app_viewer: unauthenticated".to_string()))?; + let mut tx = user_db.clone().begin(authed).await?; + let is_visible = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)", + app_id, + w_id + ) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + if !is_visible.unwrap_or(false) { + return Err(Error::NotAuthorized( + "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(), + )); + } + Ok(()) +} + #[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct PolicyTriggerableInputs { static_inputs: StaticFields, @@ -334,6 +497,13 @@ pub struct S3Key { #[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct Policy { pub on_behalf_of: Option, + /// The address `on_behalf_of` resolves to. Every write stores what the principal resolves + /// to, so it is not taken from the request except when a client names only the address — + /// which is how a cross-workspace deploy carries an identity — and it is rejected when the + /// two disagree. Optional: a policy without it executes by deriving from the principal, so + /// removing it is a change of default rather than of behavior — see + /// `docs/app-policy-email-removal.md`. + #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, //paths: // - script/ @@ -1230,29 +1400,15 @@ async fn get_public_app_by_secret( let policy = serde_json::from_str::(app.policy.0.get()).map_err(to_anyhow)?; - if !matches!(policy.execution_mode(), ExecutionMode::Anonymous) { - if opt_authed.is_none() { - return Err(Error::NotAuthorized( - "App visibility does not allow public access and you are not logged in".to_string(), - )); - } else { - let authed = opt_authed.unwrap(); - let mut tx = user_db.begin(&authed).await?; - let is_visible = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)", - id, - &w_id - ) - .fetch_one(&mut *tx) - .await?; - tx.commit().await?; - if !is_visible.unwrap_or(false) { - return Err(Error::NotAuthorized( - "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(), - )); - } - } - } + authorize_app_viewer( + policy.execution_mode(), + &app.path, + id, + &w_id, + &user_db, + &opt_authed, + ) + .await?; // Compute bundle_secret for raw apps if app.raw_app { @@ -1368,9 +1524,18 @@ async fn mint_raw_app_sdk_token( ensure_scopes_within_caller(authed, Some(scopes))?; let mut scopes = scopes.to_vec(); scopes.push(windmill_api_auth::scopes::RAW_APP_SDK_SENTINEL.to_string()); - let expiration = chrono::Utc::now() + chrono::Duration::hours(APP_EMBED_TOKEN_VALIDITY_HOURS); + let requested_exp = + chrono::Utc::now() + chrono::Duration::hours(APP_EMBED_TOKEN_VALIDITY_HOURS); + let (label, expiration) = + match guest_derived_token_constraints(db, authed, requested_exp).await? { + Some((label, exp)) => { + scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string()); + (label, exp) + } + None => (format!("sdk_app:{app_path}"), requested_exp), + }; let token_config = NewToken::new( - Some(format!("sdk_app:{app_path}")), + Some(label), Some(expiration), None, Some(scopes), @@ -1385,6 +1550,43 @@ async fn mint_raw_app_sdk_token( Ok((token, expiration)) } +/// Label and expiry a token minted *by* a guest session must carry, or `None` for a +/// non-guest minter. The label is what lets it resolve; the caller pushes the `guest` +/// sentinel so every guest control still applies; the expiry is capped at the parent's, +/// since that expiry is a guest's only revocation short of logging out. +async fn guest_derived_token_constraints( + db: &DB, + authed: &ApiAuthed, + requested: chrono::DateTime, +) -> Result)>> { + if !windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) { + return Ok(None); + } + // A guest JWT carries its own expiry and has no token row to look up; a signed-in + // guest session is a row found by prefix (MIN is the conservative side of a + // theoretical prefix collision). Either way the derived token caps on it, never on + // a fresh interval. + let parent_exp = if let Some(exp) = authed.credential_expiry { + exp + } else { + let parent: Option>> = sqlx::query_scalar( + "SELECT MIN(expiration) FROM token WHERE token_prefix = $1 AND email = $2 AND label = $3", + ) + .bind(authed.token_prefix.as_deref().unwrap_or("")) + .bind(&authed.email) + .bind(windmill_common::auth::GUEST_SESSION_LABEL) + .fetch_optional(db) + .await?; + parent.flatten().ok_or_else(|| { + Error::NotAuthorized("guest session not found or has no expiry".to_string()) + })? + }; + Ok(Some(( + windmill_common::auth::GUEST_SESSION_LABEL.to_string(), + requested.min(parent_exp), + ))) +} + /// Shared tail of the three embed-token endpoints: which credential the viewer /// gets. Sandboxed low-code gets the embed token; a sandboxed raw app declaring /// `frontend_sdk_scopes` gets the SDK token once `sdk_consent` is set — the @@ -1412,7 +1614,22 @@ pub async fn build_embed_token_response( && opt_authed.is_some() && !policy.frontend_sdk_scopes.is_empty() { - Some(policy.frontend_sdk_scopes.clone()) + // An SDK token runs as the viewer, and a guest's session is the ceiling on what + // it may delegate — the mint enforces that. Advertise only what a guest can + // actually be granted, so the consent prompt never promises a scope the mint + // would then refuse. + let declared = policy.frontend_sdk_scopes.clone(); + let offered = match opt_authed { + Some(a) if windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref()) => { + let held = a.scopes.as_deref().unwrap_or_default(); + declared + .into_iter() + .filter(|sc| held.iter().any(|h| h == sc)) + .collect::>() + } + _ => declared, + }; + (!offered.is_empty()).then_some(offered) } else { None }; @@ -1568,9 +1785,13 @@ pub async fn mint_app_embed_token( "App embed tokens cannot mint or renew embed tokens".to_string(), )); } - let expiration = + let requested_exp = chrono::Utc::now() + chrono::Duration::hours(APP_EMBED_TOKEN_VALIDITY_HOURS); - let mut scopes: Vec = APP_EMBED_SCOPES.iter().map(|s| s.to_string()).collect(); + let mut scopes: Vec = APP_EMBED_SCOPES + .iter() + .filter(|s| **s != windmill_api_auth::scopes::APP_EMBED_SENTINEL) + .map(|s| s.to_string()) + .collect(); // Path-scoped read so the app can fetch its OWN definition (apps/get/p, // which the in-workspace sandboxed viewer uses) — but no other app's. The // public viewer fetches via apps_u/public_app and doesn't rely on this. @@ -1581,10 +1802,22 @@ pub async fn mint_app_embed_token( scopes.push(format!("apps:run:{app_path}")); // A scope-restricted caller token must not bootstrap a broader-scoped // embed token (`create_token_internal` deliberately does not check this - // itself). No-op for unscoped sessions — the normal embed flow. + // itself). Checked on the real scopes only: a sentinel is a one-part string + // that `ScopeDefinition::from_scope_string` rejects, so leaving it in the + // requested set makes this fail outright for any scoped caller — which a + // guest session is. `mint_raw_app_sdk_token` has the same shape. ensure_scopes_within_caller(authed, Some(&scopes))?; + scopes.push(windmill_api_auth::scopes::APP_EMBED_SENTINEL.to_string()); + let (label, expiration) = + match guest_derived_token_constraints(db, authed, requested_exp).await? { + Some((label, exp)) => { + scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string()); + (label, exp) + } + None => (format!("embed_app:{app_path}"), requested_exp), + }; let token_config = NewToken::new( - Some(format!("embed_app:{app_path}")), + Some(label), Some(expiration), None, Some(scopes), @@ -1613,6 +1846,40 @@ pub async fn mint_app_embed_token( }) } +#[derive(Serialize)] +pub struct GuestEntry { + /// The workspace and app path to name when starting a guest sign-in. The + /// workspace is redundant on the secret route and load-bearing on the custom-path + /// one, which may not carry it in its URL. + pub workspace_id: String, + pub app_path: String, +} + +/// Whether the app behind this share secret admits guests, and under what path. +/// +/// Unauthenticated on purpose: it is what a signed-out visitor reads to learn that +/// signing in would get them in. It discloses only the app's path, to a caller who +/// already holds the share secret — the secret is the capability here. A 404 when the +/// app is not open to guests, so it says nothing about apps that are not. +async fn get_guest_entry( + Extension(db): Extension, + Path((w_id, secret)): Path<(String, String)>, +) -> JsonResult { + let id = get_id_from_secret(&db, &w_id, secret, None).await?; + let app = sqlx::query!( + "SELECT path FROM app WHERE id = $1 AND workspace_id = $2", + id, + &w_id + ) + .fetch_optional(&db) + .await?; + let app = not_found_if_none(app, "App", id.to_string())?; + if !windmill_common::workspaces::guest_app_admits(&db, &w_id, &app.path).await? { + return Err(Error::NotFound("App is not open to guests".to_string())); + } + Ok(Json(GuestEntry { workspace_id: w_id, app_path: app.path })) +} + /// Issue an embed token for a public app addressed by its (secret) share id. /// Mirrors the access check in [`get_public_app_by_secret`]: anonymous apps are /// reachable without auth, otherwise the caller must be logged in and have read @@ -1658,29 +1925,18 @@ async fn get_app_embed_token( let authed_for_token = if policy.anonymous_execution { // Anonymous app: still mint a scoped token if the viewer happens to be - // logged in (so the app sees their identity), otherwise stay anonymous. - opt_authed + // logged in (so the app sees their identity), otherwise stay anonymous. A + // guest's session names another app and cannot contain this one's scopes, + // so it renders anonymously here rather than being refused. + opt_authed.filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) } else { - let authed = opt_authed.ok_or_else(|| { - Error::NotAuthorized( - "App visibility does not allow public access and you are not logged in".to_string(), - ) - })?; - let mut tx = user_db.begin(&authed).await?; - let is_visible = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)", - id, - &w_id - ) - .fetch_one(&mut *tx) - .await?; - tx.commit().await?; - if !is_visible.unwrap_or(false) { - return Err(Error::NotAuthorized( - "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(), - )); - } - Some(authed) + let mode = if policy.guest_execution { + ExecutionMode::Guest + } else { + ExecutionMode::Publisher + }; + authorize_app_viewer(mode, &app.path, id, &w_id, &user_db, &opt_authed).await?; + opt_authed }; let resp = build_embed_token_response( @@ -1706,6 +1962,9 @@ async fn get_app_embed_token( /// strictest access interpretation. pub struct EmbedPolicyView { pub anonymous_execution: bool, + /// Open to anyone the identity provider authenticates. Like + /// `anonymous_execution`, an unknown mode reads as `false` — the strict side. + pub guest_execution: bool, pub sandbox: bool, /// Raw apps: author-declared scopes for the frontend SDK token; empty when /// the app doesn't use the frontend SDK (non-string entries are ignored). @@ -1716,6 +1975,7 @@ pub fn parse_embed_policy(policy_str: &str) -> Result { let v: serde_json::Value = serde_json::from_str(policy_str).map_err(to_anyhow)?; Ok(EmbedPolicyView { anonymous_execution: v.get("execution_mode").and_then(|m| m.as_str()) == Some("anonymous"), + guest_execution: v.get("execution_mode").and_then(|m| m.as_str()) == Some("guest"), sandbox: v.get("sandbox").and_then(|b| b.as_bool()).unwrap_or(false), frontend_sdk_scopes: v .get("frontend_sdk_scopes") @@ -2220,31 +2480,37 @@ async fn create_app_internal<'a>( } // Resolve the on-behalf-of defaults on the (non-RLS) pool *before* opening // the RLS transaction below: doing these lookups mid-transaction would hold - // a second simultaneous connection while `tx` is still checked out. + // a second simultaneous connection while `tx` is still checked out. The race this + // leaves with a concurrent rename or removal, including a freed username later + // rebinding the stored principal, is known and accepted: see `resolve_on_behalf_of`. let should_preserve = app.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed) - && app.policy.on_behalf_of.is_some(); + && (app.policy.on_behalf_of.is_some() || app.policy.on_behalf_of_email.is_some()); - if !should_preserve { + let mut preserved_on_behalf_of: Option = None; + if should_preserve { + app.policy.on_behalf_of = windmill_common::resolve_on_behalf_of( + app.policy.on_behalf_of_email.as_deref(), + app.policy.on_behalf_of.as_deref(), + true, + &authed, + w_id, + &db, + ) + .await?; + } else { let folder_default = if windmill_common::can_preserve_on_behalf_of(&authed) { windmill_common::folders::resolve_folder_default_permissioned_as(&db, w_id, &app.path) .await? } else { None }; - if let Some(default_permissioned_as) = folder_default { - let default_email = windmill_common::users::get_email_from_permissioned_as( - &default_permissioned_as, - w_id, - &db, - ) - .await?; - app.policy.on_behalf_of = Some(default_permissioned_as); - app.policy.on_behalf_of_email = Some(default_email); - } else { - app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - app.policy.on_behalf_of_email = Some(authed.email.clone()); - } + app.policy.on_behalf_of = + Some(folder_default.unwrap_or_else(|| username_to_permissioned_as(&authed.username))); + } + app.policy.on_behalf_of_email = stored_on_behalf_of_email(&app.policy, w_id, &db).await?; + if should_preserve { + preserved_on_behalf_of = audited_on_behalf_of(&app.policy, &authed); } // Reject a forged superadmin run identity in the (possibly preserved) policy. @@ -2299,10 +2565,12 @@ async fn create_app_internal<'a>( // Pin the mode the app is created under, so the stored policy states one // even when the caller did not. app.policy.set_execution_mode(app.policy.execution_mode()); - if matches!(app.policy.execution_mode(), ExecutionMode::Anonymous) { + refuse_unscopable_guest_app(&app.path, app.policy.execution_mode())?; + refuse_guest_mode_where_unavailable(&app.path, app.policy.execution_mode(), None)?; + if let Some(rule) = deployment_rule_for_mode(app.policy.execution_mode()) { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, - &ProtectionRuleKind::RestrictAnonymousAppDeployment, + &rule, &authed.username, &authed.groups, authed.is_admin, @@ -2378,21 +2646,17 @@ async fn create_app_internal<'a>( None, ) .await?; - if should_preserve { - if let Some(ref obo_email) = app.policy.on_behalf_of_email { - if obo_email != &authed.email { - audit_log( - &mut *tx, - &authed, - "apps.on_behalf_of", - ActionKind::Create, - w_id, - Some(&app.path), - Some([("on_behalf_of", obo_email.as_str()), ("action", "create")].into()), - ) - .await?; - } - } + if let Some(ref obo_email) = preserved_on_behalf_of { + audit_log( + &mut *tx, + &authed, + "apps.on_behalf_of", + ActionKind::Create, + w_id, + Some(&app.path), + Some([("on_behalf_of", obo_email.as_str()), ("action", "create")].into()), + ) + .await?; } let mut args: HashMap> = HashMap::new(); if let Some(dm) = &app.deployment_message { @@ -3149,19 +3413,33 @@ async fn update_app_internal<'a>( } } - // Reject a forged superadmin run identity in a preserved policy. Mirror the - // `should_preserve` gate below (only a preserved value is caller-controlled; - // otherwise the policy is rewritten to the deployer's own identity) and run - // it on the non-RLS pool before the transaction to avoid a second connection. - if let Some(npolicy) = ns.policy.as_ref() { + // Resolved on the (non-RLS) pool before the RLS transaction opens, for the reason + // `create_app` states, with the same known, accepted rename race (see + // `resolve_on_behalf_of`). Submitting a policy is how a deployer claims the app's execution + // identity; a source deploy that sent none claims nothing, so whoever the app already runs as + // stays. + let mut preserved_on_behalf_of: Option = None; + if let Some(npolicy) = ns.policy.as_mut() { let should_preserve = ns.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed) - && npolicy.on_behalf_of.is_some(); + && (npolicy.on_behalf_of.is_some() || npolicy.on_behalf_of_email.is_some()); + if should_preserve { - windmill_common::auth::validate_on_behalf_of( - npolicy.on_behalf_of.as_deref(), + npolicy.on_behalf_of = windmill_common::resolve_on_behalf_of( npolicy.on_behalf_of_email.as_deref(), - )?; + npolicy.on_behalf_of.as_deref(), + true, + &authed, + w_id, + &db, + ) + .await?; + } else { + npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); + } + npolicy.on_behalf_of_email = stored_on_behalf_of_email(npolicy, w_id, &db).await?; + if should_preserve { + preserved_on_behalf_of = audited_on_behalf_of(npolicy, &authed); } } @@ -3194,7 +3472,6 @@ async fn update_app_internal<'a>( reject_kind_change(path, raw_app, deployed_raw_app)?; } - let mut preserved_on_behalf_of: Option = None; let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() @@ -3213,6 +3490,28 @@ async fn update_app_internal<'a>( if npath != path { require_owner_of_path(&authed, path)?; + // The destination is what a guest session would be scoped to. A rename + // that carries no policy keeps the deployed mode, read under the row + // lock so a policy update landing alongside cannot slip a guest app + // onto a path it cannot be scoped to. + let mode = match ns.policy.as_ref().and_then(|p| p.stated_execution_mode()) { + Some(mode) => mode, + None => sqlx::query_scalar::<_, Option>( + "SELECT policy->>'execution_mode' FROM app + WHERE path = $1 AND workspace_id = $2 FOR UPDATE", + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await? + .flatten() + .and_then(|m| { + serde_json::from_value::(serde_json::Value::String(m)).ok() + }) + .unwrap_or_default(), + }; + refuse_unscopable_guest_app(npath, mode)?; + let exists = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)", npath, @@ -3320,21 +3619,33 @@ async fn update_app_internal<'a>( .unwrap_or_default(), ); } - if matches!(npolicy.execution_mode(), ExecutionMode::Anonymous) && !authed.is_admin { - // Restricted users may keep deploying an app that is already - // public, but flipping an app to anonymous (public) access is - // gated by the RestrictAnonymousAppDeployment protection rule. - // An unreadable deployed policy reads as not-anonymous, the - // strict direction. - let already_anonymous = deployed + refuse_unscopable_guest_app( + ns.path.as_deref().unwrap_or(path), + npolicy.execution_mode(), + )?; + // An unreadable deployed policy reads as not already-in-mode, the strict + // direction, as for the protection rule below. + refuse_guest_mode_where_unavailable( + ns.path.as_deref().unwrap_or(path), + npolicy.execution_mode(), + deployed_policy.as_ref().map(|d| d.execution_mode()), + )?; + if let Some(rule) = + deployment_rule_for_mode(npolicy.execution_mode()).filter(|_| !authed.is_admin) + { + // Restricted users may keep deploying an app that is already open + // to this audience, but widening one is gated by the matching + // protection rule. An unreadable deployed policy reads as not + // already-widened, the strict direction. + let already_in_mode = deployed .as_ref() .and_then(|p| p.get("execution_mode")) .and_then(|m| m.as_str()) - == Some("anonymous"); - if !already_anonymous { + == Some(npolicy.execution_mode().as_str()); + if !already_in_mode { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, - &ProtectionRuleKind::RestrictAnonymousAppDeployment, + &rule, &authed.username, &authed.groups, authed.is_admin, @@ -3346,23 +3657,6 @@ async fn update_app_internal<'a>( } } } - let should_preserve = ns.preserve_on_behalf_of.unwrap_or(false) - && windmill_common::can_preserve_on_behalf_of(&authed) - && npolicy.on_behalf_of.is_some(); - - if should_preserve { - if let Some(ref obo_email) = npolicy.on_behalf_of_email { - if obo_email != &authed.email { - preserved_on_behalf_of = Some(obo_email.clone()); - } - } - } else if caller_sent_policy { - // Submitting a policy is how a deployer claims the app's - // execution identity. A source deploy that sent none is not - // claiming anything, so whoever the app already runs as stays. - npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - npolicy.on_behalf_of_email = Some(authed.email.clone()); - } sqlb.set( "policy", quote(serde_json::to_string(&json!(npolicy)).map_err(|e| { @@ -3585,17 +3879,36 @@ fn digest(code: &str) -> String { async fn get_on_behalf_details_from_policy_and_authed( policy: &Policy, opt_authed: &Option, + w_id: &str, + db: &DB, ) -> Result<(String, String, String)> { + // A guest acts only through an app open to guests — or to everyone. A members-only + // mode means the policy changed after the session was issued. Decided here, in the + // one resolver every on-behalf path (runs, S3 reads, uploads) goes through. + if opt_authed + .as_ref() + .is_some_and(|a| windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + && !matches!( + policy.execution_mode(), + ExecutionMode::Guest | ExecutionMode::Anonymous + ) + { + return Err(Error::PermissionDenied( + "this app is not open to guests".to_string(), + )); + } let (username, permissioned_as, email) = match policy.execution_mode() { ExecutionMode::Anonymous => { let username = opt_authed .as_ref() .map(|a| a.username.clone()) .unwrap_or_else(|| "anonymous".to_string()); - let (permissioned_as, email) = get_on_behalf_of(&policy)?; + let (permissioned_as, email) = get_on_behalf_of(&policy, w_id, db).await?; (username, permissioned_as, email) } - ExecutionMode::Publisher => { + // Guest runs as the publisher exactly as Publisher does; the two differ only + // in who is let through the door, which is settled before we get here. + ExecutionMode::Publisher | ExecutionMode::Guest => { let username = opt_authed .as_ref() .map(|a| a.username.clone()) @@ -3604,7 +3917,7 @@ async fn get_on_behalf_details_from_policy_and_authed( "publisher execution mode requires authentication".to_string(), ) })?; - let (permissioned_as, email) = get_on_behalf_of(&policy)?; + let (permissioned_as, email) = get_on_behalf_of(&policy, w_id, db).await?; (username, permissioned_as, email) } ExecutionMode::Viewer => { @@ -3680,8 +3993,12 @@ async fn execute_component( // Authorize before touching the payload: the route layer is resource-blind, so a // path-scoped caller (app embed token, or a picker-minted `apps:run|write:`) // is confined to its own app only here. No-op for unscoped callers; anonymous ones - // are policy-gated below. - if let Some(authed) = opt_authed.as_ref() { + // are policy-gated below, and a guest's confinement waits for the app's mode + // (`guest_caller_for_mode`). + if let Some(authed) = opt_authed + .as_ref() + .filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + { check_scopes(authed, || format!("apps:run:{}", path))?; } // Only honor temp_script_refs for the inline-script preview path: @@ -3898,8 +4215,16 @@ async fn execute_component( } }; - // Check rate limit for anonymous (public) executions - if matches!(policy.execution_mode(), ExecutionMode::Anonymous) && opt_authed.is_none() { + // Rate limit for executions by callers the workspace does not know: anonymous + // viewers, and guests — on an instance whose provider accepts any consumer + // account, "anyone the IdP authenticates" is close to the anonymous population, + // and each run costs a job as the publisher. + let is_guest_caller = opt_authed + .as_ref() + .is_some_and(|a| windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())); + if (matches!(policy.execution_mode(), ExecutionMode::Anonymous) && opt_authed.is_none()) + || is_guest_caller + { if let Some(limit) = crate::workspaces::get_public_app_rate_limit(&db, &w_id).await? { if limit > 0 { crate::public_app_rate_limit::check_and_increment(&w_id, limit)?; @@ -3907,6 +4232,8 @@ async fn execute_component( } } + let opt_authed = guest_caller_for_mode(opt_authed, policy.execution_mode(), path)?; + // Execution is publisher and an user is authenticated: check if the user is authorized to // execute the app. if let (ExecutionMode::Publisher, Some(authed)) = (policy.execution_mode(), opt_authed.as_ref()) @@ -3948,7 +4275,7 @@ async fn execute_component( } let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); @@ -4242,8 +4569,11 @@ async fn upload_s3_file_from_app( request: axum::extract::Request, ) -> JsonResult { // Same path confinement as `execute_component`: without it a token scoped to app A - // could drive app B's upload policy. - if let Some(authed) = opt_authed.as_ref() { + // could drive app B's upload policy. A guest's waits for the app's mode, below. + if let Some(authed) = opt_authed + .as_ref() + .filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + { check_scopes(authed, || format!("apps:run:{}", path.to_path()))?; } let policy = if let Some(file_key_regex) = query.force_viewer_file_key_regex { @@ -4296,6 +4626,14 @@ async fn upload_s3_file_from_app( .map(|p| serde_json::from_value::(p).map_err(to_anyhow)) .transpose()? }; + let opt_authed = guest_caller_for_mode( + opt_authed, + policy + .as_ref() + .map(Policy::execution_mode) + .unwrap_or_default(), + path.to_path(), + )?; let user_db = UserDB::new(db.clone()); @@ -4307,7 +4645,7 @@ async fn upload_s3_file_from_app( let s3_inputs = policy.s3_inputs.as_ref().unwrap(); let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let on_behalf_authed = fetch_api_authed_from_permissioned_as( permissioned_as.clone(), @@ -4453,8 +4791,12 @@ async fn upload_s3_file_from_app( } } else { // backward compatibility (no policy) - // if no policy but logged in, use the user's auth to get the s3 resource - if let Some(authed) = opt_authed { + // if no policy but logged in, use the user's auth to get the s3 resource. A guest + // has no standing of its own to upload with, so without a policy it is refused + // exactly as an anonymous caller is. + if let Some(authed) = opt_authed + .filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + { let file_key = query .file_key .unwrap_or_else(|| get_random_file_name(query.file_extension)); @@ -4712,8 +5054,9 @@ async fn get_on_behalf_authed_from_app( }) }; + let opt_authed = guest_caller_for_mode(opt_authed.clone(), policy.execution_mode(), path)?; let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let on_behalf_authed = fetch_api_authed_from_permissioned_as(permissioned_as, email, &w_id, &db, Some(username)) @@ -4868,6 +5211,10 @@ fn check_app_s3_read_scope(opt_authed: &Option, path: &str) -> Result let Some(authed) = opt_authed.as_ref() else { return Ok(()); }; + // A guest's confinement waits for the app's mode (`get_on_behalf_authed_from_app`). + if windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) { + return Ok(()); + } check_scopes(authed, || format!("apps:run:{}", path)) .or_else(|_| check_scopes(authed, || format!("apps:read:{}", path))) } @@ -5220,7 +5567,45 @@ async fn app_load_csv_preview() -> Result<()> { )) } -fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { +/// The address to store beside the principal. Derived from it, never taken from the request, so +/// the stored copy can only ever agree with the principal — the drift it used to allow is what +/// this replaces. +/// +/// Written unconditionally, including for the versions that could derive it instead: a replica +/// predating that fallback fails outright when the key is absent, which would 400 every +/// anonymous, publisher and guest app for the length of a rolling deploy. The write is what +/// holds the key in place — see `docs/app-policy-email-removal.md`. +async fn stored_on_behalf_of_email(policy: &Policy, w_id: &str, db: &DB) -> Result> { + let Some(permissioned_as) = policy.on_behalf_of.as_deref() else { + return Ok(None); + }; + Ok(Some( + windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) + .await?, + )) +} + +/// The address to record in the `apps.on_behalf_of` audit entry: the one the app will run as, +/// when it is not the deployer's own. `None` when they match — a deployer handing an app their +/// own identity is not an on-behalf-of deploy. +/// +/// Reads the address `stored_on_behalf_of_email` just resolved rather than looking it up again, +/// so the audit row and the policy row can only ever name the same account. +fn audited_on_behalf_of(policy: &Policy, authed: &ApiAuthed) -> Option { + policy + .on_behalf_of_email + .as_deref() + .filter(|email| *email != authed.email) + .map(str::to_string) +} + +/// The identity an anonymous, publisher or guest execution runs as. +/// +/// `on_behalf_of_email` is optional: every write stores it, so it is present on anything this +/// release deployed, and it is only derived for a policy that predates that. Deriving is the +/// fallback rather than the rule so that removing the key later is a change of default, not a +/// change of behavior — see `docs/app-policy-email-removal.md`. +async fn get_on_behalf_of(policy: &Policy, w_id: &str, db: &DB) -> Result<(String, String)> { let permissioned_as = policy .on_behalf_of .as_ref() @@ -5231,16 +5616,15 @@ fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { ) })? .to_string(); - let email = policy - .on_behalf_of_email - .as_ref() - .ok_or_else(|| { - Error::BadRequest( - "on_behalf_of_email is missing in the app policy and is required for anonymous execution" - .to_string(), - ) - })? - .to_string(); + let email = match policy.on_behalf_of_email.as_deref() { + Some(email) => email.to_string(), + // Cached on purpose, up to one notify poll stale: the accepted dispatch case + // `get_email_from_permissioned_as` documents. + None => { + windmill_common::users::get_email_from_permissioned_as(&permissioned_as, w_id, db) + .await? + } + }; // Defence in depth against a policy that already carries a forged superadmin // sentinel (deployed before validation existed, or copied verbatim by a // workspace fork): the sentinels are internal-only and never a legitimate app @@ -5387,7 +5771,25 @@ async fn build_args( "email" => authed.as_ref().map(|a| serde_json::to_value(&a.email)), "workspace" => Some(serde_json::to_value(&w_id)), "groups" => authed.as_ref().map(|a| serde_json::to_value(&a.groups)), - "author" => Some(serde_json::to_value(&policy.on_behalf_of_email)), + // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. + "author" => { + let author = match ( + policy.on_behalf_of_email.as_deref(), + policy.on_behalf_of.as_deref(), + ) { + (Some(email), _) => Some(email.to_string()), + (None, Some(permissioned_as)) => Some( + windmill_common::users::get_email_from_permissioned_as( + permissioned_as, + w_id, + db, + ) + .await?, + ), + (None, None) => None, + }; + Some(serde_json::to_value(&author)) + } _ => { return Err(Error::BadRequest(format!( "context variable {} not allowed", @@ -5541,6 +5943,9 @@ mod embed_token_tests { "GET", ), ("/api/w/test/resources/list_search", "GET"), + // The metadata allowlist matches `resources/type/` by prefix, so a route + // added under it that is not a read must be denied by its method. + ("/api/w/test/resources/type/hub/pick/slack", "POST"), // Workspace-wide job enumeration/export must NOT be reachable — an app // reads only jobs it launched, by id (blocked via the app_embed sentinel). ("/api/w/test/jobs/list", "GET"), diff --git a/backend/windmill-api/src/csrf.rs b/backend/windmill-api/src/csrf.rs new file mode 100644 index 0000000000..4b2915f791 --- /dev/null +++ b/backend/windmill-api/src/csrf.rs @@ -0,0 +1,251 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::extract::FromRequestParts; +use axum::http::{header, request::Parts, Method}; +use axum::Extension; +use url::Url; +use windmill_common::error::Error; +use windmill_common::users::COOKIE_NAME; + +use crate::triggers::trigger_helpers::RunnableId; + +/// Whether a request is a cross-site GET authenticating on the session cookie alone. A GET +/// handler that runs a script by path resolves it through [`Self::script_runnable`], which +/// refuses a Hub script on such a request. +/// +/// The cookie is `SameSite=Lax`, so browsers attach it to cross-site top-level GET +/// navigations. A `hub/` path runs any public Hub script, and an argument written +/// `$var:` or `$res:` is resolved as the caller before the script sees it: such a +/// GET lets any page pick a generic Hub script and hand it the victim's secrets, which the job +/// can then send anywhere. +/// +/// Workspace scripts and flows are not refused, by choice, so that GET links to them keep +/// working. That is a scope decision, not a safety property: they still take attacker-chosen +/// arguments, `$var:` and `$res:` included, resolved as the victim. What bounds the exposure +/// is that the attacker needs a runnable path and can only run code the workspace deployed. +/// +/// The cookie is the only ambient credential. A bearer header is explicit, and so is the +/// `token` query parameter the webhook URLs carry — a cross-origin `EventSource` has no other +/// way to authenticate, since it cannot set headers. The checks run in `extract_token`'s +/// order, header before cookie, because that is the order it resolves them in: a request +/// carrying both a cookie and `token=` authenticates on the cookie and is therefore still +/// ambient, which is also why a valid `token=` link opened cross-site while signed in is +/// refused. +pub struct CrossSiteGetGuard(Option); + +impl CrossSiteGetGuard { + pub fn script_runnable(&self, script_path: &str) -> windmill_common::error::Result { + let runnable_id = RunnableId::from_script_path(script_path); + let (Some(signal), RunnableId::HubScript(_)) = (&self.0, &runnable_id) else { + return Ok(runnable_id); + }; + // The `Referer` leg is the one that can misfire, on a request that really was + // same-host: it compares against the hosts the backend can see, and a proxy that + // rewrites `Host` without setting `X-Forwarded-Host` leaves none of them matching + // what the browser addressed. Name the comparison so that shows up as a + // misconfiguration rather than as an unexplained 403. + if let CrossSite::RefererMismatch { referer, instance_hosts } = signal { + tracing::warn!( + referer_host = %referer, + ?instance_hosts, + "refusing a cross-site GET Hub script run inferred from Referer; if the request \ + was same-host, set `X-Forwarded-Host` on the proxy or configure `BASE_URL`" + ); + } + Err(Error::PermissionDenied( + "a cross-site GET request cannot run a Hub script with the session cookie, which takes \ + precedence over a `token` query parameter: pass the token in the `Authorization` \ + header, or open the link from the instance itself or from a browser with no Windmill \ + session" + .to_string(), + )) + } +} + +impl FromRequestParts for CrossSiteGetGuard { + type Rejection = std::convert::Infallible; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method != Method::GET { + return Ok(CrossSiteGetGuard(None)); + } + let Some(signal) = cross_site_signal(parts) else { + return Ok(CrossSiteGetGuard(None)); + }; + + let has_bearer = parts + .headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.starts_with("Bearer ")); + if has_bearer { + return Ok(CrossSiteGetGuard(None)); + } + + let has_session_cookie = + Extension::::from_request_parts(parts, state) + .await + .is_ok_and(|Extension(cookies)| cookies.get(COOKIE_NAME).is_some()); + Ok(CrossSiteGetGuard(has_session_cookie.then_some(signal))) + } +} + +enum CrossSite { + Declared, + RefererMismatch { referer: String, instance_hosts: Vec }, +} + +fn cross_site_signal(parts: &Parts) -> Option { + if let Some(site) = parts.headers.get("sec-fetch-site") { + return site + .as_bytes() + .eq_ignore_ascii_case(b"cross-site") + .then_some(CrossSite::Declared); + } + // Fetch Metadata rides only on potentially trustworthy URLs, so an instance served + // over plain http never receives `Sec-Fetch-Site` (nor does Safari before 16.4) while + // the cookie, not being `Secure` there either, still arrives. `Referer` is the only + // other thing a top-level GET navigation carries — `Origin` is not sent on one — so it + // is all that is left there, and it is weak: the default `strict-origin-when-cross- + // origin` policy already drops `Referer` on an https-to-http downgrade, so an https + // attacker page pointing a victim at a plain-http instance sends neither header. This + // leg catches an http-served attacker page and pre-16.4 Safari on https; the guard is + // load-bearing on https and best-effort at best on plain http. An absent `Referer` + // reads as not cross-site, matching how `Sec-Fetch-Site: none` (a bookmark, a typed + // URL) is treated. + let referer = referer_host(parts)?; + let instance_hosts: Vec = instance_hosts(parts).collect(); + (!instance_hosts + .iter() + .any(|host| host.eq_ignore_ascii_case(&referer))) + .then_some(CrossSite::RefererMismatch { referer, instance_hosts }) +} + +/// Every host a legitimate same-host request can name. `Host` alone is not enough: a +/// reverse proxy that forwards without preserving it (nginx `proxy_pass` with no +/// `proxy_set_header Host $host`) hands the backend the upstream's name, which no browser +/// `Referer` will ever match. None of these is browser-settable on a navigation — a +/// navigation carries no custom headers, and `BASE_URL` is instance config — so widening +/// the accepted set costs nothing. +fn instance_hosts(parts: &Parts) -> impl Iterator { + let base_url = windmill_common::BASE_URL.load(); + [ + request_host(parts), + header_host(parts, "x-forwarded-host"), + Url::parse(base_url.as_str()) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)), + ] + .into_iter() + .flatten() +} + +fn referer_host(parts: &Parts) -> Option { + let referer = parts.headers.get(header::REFERER)?.to_str().ok()?; + Url::parse(referer).ok()?.host_str().map(str::to_owned) +} + +fn request_host(parts: &Parts) -> Option { + if let Some(host) = parts.uri.host() { + return Some(host.to_owned()); + } + header_host(parts, header::HOST) +} + +fn header_host(parts: &Parts, name: impl header::AsHeaderName) -> Option { + host_of(parts.headers.get(name)?.to_str().ok()?) +} + +/// The host in a `Host`-shaped header value: `host[:port]`, where `host` may be a bracketed +/// IPv6 literal, and where a chain of proxies appends to `X-Forwarded-Host` so only the +/// first entry is the one the browser addressed. The port is split off by the URL parser +/// rather than by hand-rolling the bracket rules. +fn host_of(value: &str) -> Option { + let host = value.split(',').next()?.trim(); + Url::parse(&format!("http://{host}")) + .ok()? + .host_str() + .map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::{cross_site_signal, host_of, CrossSite}; + use axum::http::{request::Parts, Request}; + + fn parts(headers: &[(&str, &str)]) -> Parts { + let mut req = Request::get("/api/w/ws/jobs/run_wait_result/p/hub/1/x"); + for (name, value) in headers { + req = req.header(*name, *value); + } + req.body(()).unwrap().into_parts().0 + } + + #[test] + fn sec_fetch_site_decides_when_present() { + let declared = |site| cross_site_signal(&parts(&[("sec-fetch-site", site)])); + assert!(matches!(declared("cross-site"), Some(CrossSite::Declared))); + for site in ["same-origin", "same-site", "none"] { + assert!(declared(site).is_none(), "{site} is not cross-site"); + } + // The header outranks a `Referer` that disagrees with it. + let with_referer = parts(&[ + ("sec-fetch-site", "same-origin"), + ("host", "windmill.example"), + ("referer", "https://attacker.example/page"), + ]); + assert!(cross_site_signal(&with_referer).is_none()); + } + + #[test] + fn referer_stands_in_when_sec_fetch_site_is_absent() { + let signal = |headers: &[(&str, &str)]| cross_site_signal(&parts(headers)); + assert!(matches!( + signal(&[ + ("host", "windmill.example"), + ("referer", "https://attacker.example/p") + ]), + Some(CrossSite::RefererMismatch { .. }) + )); + // Ports differ between the frontend and the API, and do not make a request cross-site. + assert!(signal(&[ + ("host", "windmill.example:8000"), + ("referer", "http://windmill.example:3000/apps"), + ]) + .is_none()); + // A proxy that rewrote `Host` but forwarded the public name. + assert!(signal(&[ + ("host", "windmill-server.internal"), + ("x-forwarded-host", "windmill.example"), + ("referer", "https://windmill.example/apps"), + ]) + .is_none()); + assert!(signal(&[("host", "windmill.example")]).is_none()); + } + + #[test] + fn host_of_strips_port_brackets_and_proxy_chain() { + assert_eq!(host_of("windmill.example"), Some("windmill.example".into())); + assert_eq!( + host_of("windmill.example:8000"), + Some("windmill.example".into()) + ); + assert_eq!(host_of("[::1]:8000"), Some("[::1]".into())); + assert_eq!(host_of("[::1]"), Some("[::1]".into())); + assert_eq!( + host_of("windmill.example, proxy.internal"), + Some("windmill.example".into()) + ); + assert_eq!(host_of(""), None); + assert_eq!(host_of("not a host"), None); + } +} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index d781e229d2..e8a6858db7 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -108,6 +108,9 @@ lazy_static::lazy_static! { (20260826214706, include_str!( "../../migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql" ).replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260909163047, include_str!( + "../../migrations/20260909163047_workspace_delete_cascade_indexes.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } diff --git a/backend/windmill-api/src/git_sync_oss.rs b/backend/windmill-api/src/git_sync_oss.rs index 0451d88699..d1653d9aa9 100644 --- a/backend/windmill-api/src/git_sync_oss.rs +++ b/backend/windmill-api/src/git_sync_oss.rs @@ -10,6 +10,11 @@ pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] +pub fn workspaced_git_sync_service() -> Router { + Router::new() +} + #[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index ccb530ce20..633ad88a34 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -6,13 +6,14 @@ use axum::{ http::{request::Parts, StatusCode}, response::{IntoResponse, Response}, routing::{get, post}, - Router, + Extension, Router, }; use serde::{Deserialize, Deserializer, Serialize}; use windmill_common::{ error::{to_anyhow, Error}, + global_settings::{load_value_from_global_settings, DISABLE_HUB_SETTING}, utils::require_admin, - HUB_BASE_URL, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, }; pub fn workspaced_service() -> Router { @@ -48,6 +49,7 @@ pub fn workspaced_service() -> Router { post(discard_project_update), ) .route("/project", get(get_project_by_source)) + .route("/projects", get(list_projects)) } #[derive(Deserialize)] @@ -548,6 +550,84 @@ async fn get_project_by_source(ctx: HubPublishCtx) -> Result bool { + fn host_of(url: &str) -> Option { + let parsed = url::Url::parse(url.trim()).ok()?; + if !matches!(parsed.scheme(), "http" | "https") { + return None; + } + Some( + parsed + .host_str()? + .trim_end_matches('.') + .to_ascii_lowercase(), + ) + } + match (host_of(hub), host_of(DEFAULT_HUB_BASE_URL)) { + (Some(host), Some(default_host)) => host == default_host, + _ => false, + } +} + +// The hub's project catalogue. Read by any workspace member rather than through +// `HubPublishCtx`, which requires an admin: nothing here is workspace-scoped or +// publishing-related. It exists at all because the hub's listing endpoint sends no +// CORS header, so the browser cannot read it directly the way it reads a single +// project. `accept: application/json` is what makes the hub answer with JSON. +// +// The caller's token is sent only to a hub this instance was pointed at deliberately. +// Every other route here is admin-only; this one is not, so forwarding a member's +// bearer token to `hub.windmill.dev` would put a credential replayable against this +// instance on a host outside it — for a listing that needs no credential at all. +async fn list_projects( + _authed: ApiAuthed, + Extension(db): Extension, + Tokened { token }: Tokened, +) -> Result { + // `disable_hub` turns the hub off for a closed instance, and this handler makes an + // outbound request. The frontend hides its entry points on the same setting, but that + // is presentation: an authenticated member can call this route directly, so the refusal + // has to live here. + let disabled = load_value_from_global_settings(&db, DISABLE_HUB_SETTING) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if disabled { + return Err(Error::BadRequest( + "The hub is disabled on this instance".to_string(), + )); + } + + let hub = (**HUB_BASE_URL.load()).clone(); + let url = format!("{}/projects", hub); + let mut req = HTTP_CLIENT.get(&url).header("accept", "application/json"); + if !is_public_hub(&hub) { + req = req.bearer_auth(&token); + } + let res = req + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} + async fn submit_project( ctx: HubPublishCtx, Path((_workspace, slug)): Path<(String, ProjectSlug)>, @@ -645,3 +725,42 @@ async fn forward_to_hub( Ok((status, text)) } + +#[cfg(test)] +mod tests { + use super::is_public_hub; + + #[test] + fn public_hub_recognized_in_every_spelling() { + // The predicate decides whether a workspace member's bearer token leaves the + // instance, so both directions matter: a miss on the public hub sends the token + // to windmill.dev, and a false match withholds it from a private hub that needs it. + // Every spelling here is one `hub_base_url` can hold and `reqwest` will still send. + for hub in [ + "https://hub.windmill.dev", + "http://hub.windmill.dev/", + "HTTPS://hub.windmill.dev", + "https://HUB.WINDMILL.DEV", + "https://hub.windmill.dev:443", + "https://hub.windmill.dev.", + "https://hub.windmill.dev/some/path", + " https://hub.windmill.dev ", + ] { + assert!(is_public_hub(hub), "{hub} should be the public hub"); + } + for hub in [ + "https://hub.internal.example", + "https://hub.windmill.dev.evil.example", + "https://windmill.dev", + // The host is what the request goes to, whatever precedes the `@`. + "https://hub.windmill.dev@hub.internal.example", + // Unparseable, or not a scheme a request can be built from. Grouped with the + // private hubs because the caller then attaches the token, which is harmless here: + // `reqwest` rejects the same value before opening a connection. + "hub.windmill.dev", + "ftp://hub.windmill.dev", + ] { + assert!(!is_public_hub(hub), "{hub} should not be the public hub"); + } + } +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 53044ee020..84968ebbcc 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -73,6 +73,7 @@ use crate::{ args::{self, RawWebhookArgs}, auth::{OptTokened, Tokened}, concurrency_groups::join_concurrency_key, + csrf::CrossSiteGetGuard, db::{ApiAuthed, DB}, triggers::trigger_helpers::RunnableId, users::{ @@ -106,7 +107,10 @@ use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error}, flow_status::{Approval, ApprovalConditions, FlowStatus, FlowStatusModule}, - flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, + flows::{ + add_virtual_items_if_necessary, resolve_maybe_value, ApprovalSkin, FlowModule, FlowValue, + Suspend, + }, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, query_builders, @@ -139,6 +143,7 @@ pub fn workspaced_service() -> Router { .route("/run_progress/{id}", get(get_run_progress)) .route("/run_assets/{id}", get(list_run_assets)) .route("/dbt_graph/{id}", get(get_dbt_run_graph)) + .route("/dbt_column_lineage/{id}", get(get_dbt_run_column_lineage)) .route("/dbt_resumable/{id}", get(get_dbt_resumable)) .route( "/dbt_resumable_script/p/{*script_path}", @@ -891,21 +896,27 @@ struct AssetProgress { error: Option, } -/// The asset graph as one run saw it. Pinning to a job needs the full job-read -/// contract, so it lives on `require_job_read_access` here rather than as a -/// parameter on `/assets/graph`. See docs/dbt-runtime.md. -async fn get_dbt_run_graph( - authed: ApiAuthed, - OptViewToken(view_token): OptViewToken, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Query(q): Query, -) -> error::JsonResult { +/// Which project version a dbt view pins to for this job, once the caller has +/// been shown to be entitled to it. +/// +/// `Ok(None)` is "answer unpinned", not a refusal: a job that stored no graph of +/// its own — and one that has aged out of retention — is served the deployed +/// version rather than an error, so a run page keeps drawing after the run is +/// gone. Pinning needs the full job-read contract, which is why it lives on +/// `require_job_read_access` here rather than as a parameter on `/assets/*`. +/// See docs/dbt-runtime.md. +async fn dbt_pinned_run( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + job_id: Uuid, + view_token: Option<&str>, +) -> error::Result> { // The scope domain comes from the URL segment, so `/jobs` asks a scoped token // for `jobs:read` alone while the body returned is asset data. Both are // required: the job gate below reaches this run, this reaches assets at all. - check_scopes(&authed, || "assets:read".to_string())?; + check_scopes(authed, || "assets:read".to_string())?; let job = sqlx::query!( r#"SELECT created_by, runnable_path, CASE WHEN kind = 'script' THEN runnable_id END AS script_hash, @@ -918,42 +929,70 @@ async fn get_dbt_run_graph( AND g.script_hash IS NULL) AS "editor_graph!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, job_id, - &w_id + w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await?; - // No such job: answer the unpinned graph rather than 404, so a run page whose - // job has aged out of retention still draws the deployed version instead of - // an error. Reachable only with `assets:read`, which is exactly what - // `/assets/graph` would have cost for the same answer. + // Unpinned rather than 404 for a job that is gone. Reachable only with + // `assets:read`, which is exactly what the unpinned route would have cost + // for the same answer. let Some(job) = job else { - return windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, None).await; + return Ok(None); }; require_job_read_access( - &db, - &user_db, - &authed, - &w_id, + db, + user_db, + authed, + w_id, &job_id, &job.created_by, - view_token.as_deref(), + view_token, ) .await?; // A preview or flow job names no deployed version, so there is usually no // graph to pin to and the workspace one answers. The exception is a job that // parsed one itself, which is what the dbt editor's refresh is: its graph // belongs to that job alone and nothing else can reach it. - let pinned = job + Ok(job .runnable_path .filter(|_| job.script_hash.is_some() || job.editor_graph) .map(|path| windmill_api_assets::PinnedRun { job_id, script_path: path, script_hash: job.script_hash, - }); + })) +} + +/// The asset graph as one run saw it. +async fn get_dbt_run_graph( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(q): Query, +) -> error::JsonResult { + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, pinned).await } +/// The column lineage a set of relations sits in as one run saw it — the same +/// pin as `get_dbt_run_graph`, for the trace drawn beside a node of that graph. +async fn get_dbt_run_column_lineage( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(pairs): Query>, +) -> error::JsonResult { + let q = windmill_api_assets::ColumnLineageQuery::from_query_pairs(pairs)?; + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; + windmill_api_assets::dbt_column_lineage_for(&authed, &w_id, user_db, q, pinned).await +} + /// Whether a `dbt retry` submitted by this caller would resume THIS run. /// /// One failure is saved per script per execution principal, so a page showing an @@ -1594,7 +1633,11 @@ pub(crate) async fn require_job_read_access( // this token, and letting it reach any job merely visible to the viewer would // expose unrelated runs' results/logs. Stop at the launched-by-viewer grant. // NotFound (not PermissionDenied) so the untrusted app can't probe job existence. - if windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) { + // A guest stops here too: it has no membership behind it, so a share token whose + // audience is the workspace's members must not read for it either. + if windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) + || windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) + { return Err(Error::NotFound(format!("Job {job_id} not found"))); } @@ -4377,6 +4420,15 @@ async fn count_completed_jobs( )) } +lazy_static::lazy_static! { + /// 0 keeps the connection-wide statement_timeout. + static ref LIST_JOBS_STATEMENT_TIMEOUT_SECS: u64 = + std::env::var("LIST_JOBS_STATEMENT_TIMEOUT_SECS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(30); +} + async fn list_jobs( authed: ApiAuthed, Extension(user_db): Extension, @@ -4494,10 +4546,32 @@ async fn list_jobs( // tracing::info!("sql: {}", &sql); let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + // A client that gives up does not cancel its query, so without this bound every retry of a + // slow filter stacks another scan running until the connection-wide 5min timeout. + let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; + if timeout_secs > 0 { + sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'")) + .execute(&mut *tx) + .await?; + } + let jobs: Vec = sqlx::query_as(&sql) .fetch_all(&mut *tx) .warn_after_seconds_with_sql(5, format!("list_jobs: {}", sql)) - .await?; + .await + .map_err(|e| match e { + sqlx::Error::Database(ref db_err) + if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") => + { + Error::Generic( + StatusCode::BAD_REQUEST, + format!( + "Listing jobs took more than {timeout_secs}s and was stopped. Set a start date or narrow the filters." + ), + ) + } + e => e.into(), + })?; tx.commit().await?; Ok(Json(jobs.into_iter().map(From::from).collect())) @@ -4787,6 +4861,11 @@ struct ApprovalInfo { user_auth_required: bool, #[serde(skip_serializing_if = "Option::is_none")] hide_cancel: Option, + skin: ApprovalSkin, + #[serde(skip_serializing_if = "Option::is_none")] + step_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_summary: Option, approvers: Vec, /// Share-read-link token for the flow, minted only for callers allowed to view this /// approval. Lets an authenticated workspace-member approver open the run details of @@ -4840,6 +4919,48 @@ fn can_approve_step( } } +/// The latest approval step the run has passed: a step before the current `step` that ran +/// rather than being skipped. Steps from `step` on don't count, because while an approval is +/// pending the step after it already holds the `WaitingForEvents` status. +fn last_reached_approval_step<'a>( + flow: &'a FlowValue, + status: &FlowStatus, +) -> Option<&'a FlowModule> { + flow.modules + .iter() + .zip(status.modules.iter()) + .take(usize::try_from(status.step).unwrap_or(0)) + .rev() + .filter(|(_, m)| matches!(m, FlowStatusModule::Success { skipped: false, .. })) + .map(|(module, _)| module) + .find(|module| module.suspend.is_some()) +} + +/// The approval conditions a step's own settings give, as the worker records them when the step +/// suspends. The worker drops them from the run once the step is approved, so a run that has +/// moved on is gated by these. Groups computed by an expression can't be re-evaluated outside +/// the run, so such a step falls back to any signed-in user. +fn approval_conditions_from_settings(suspend: &Suspend) -> Option { + let user_auth_required = suspend.user_auth_required.unwrap_or(false); + let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false); + if !user_auth_required && !self_approval_disabled { + return None; + } + let user_groups_required = match &suspend.user_groups_required { + Some(InputTransform::Static { value }) if user_auth_required => { + serde_json::from_str(value.get()).unwrap_or_default() + } + _ => vec![], + }; + Some(ApprovalConditions { user_auth_required, user_groups_required, self_approval_disabled }) +} + +/// How the approval step presents itself on the approval page. +struct ApprovalStepView { + skin: ApprovalSkin, + summary: Option, +} + async fn get_approval_info( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -4868,13 +4989,33 @@ async fn get_approval_info( script_path: Option, email: String, flow_status: Option, - workflow_as_code_status: Option, + // `v2_job_status` only holds a run that hasn't finished, so the fields below also read + // the completed run's status: a finished run's page keeps its skin and, for workflows as + // code, its description, still gated by the approval conditions the run had. + completed_flow_status: Option, + is_wac: bool, + wac_approval: Option, + approval_conditions: Option, + flow_summary: Option, } let row = sqlx::query_as::<_, ApprovalJobRow>( "SELECT j.id, j.runnable_path as script_path, j.permissioned_as_email as email, - s.flow_status, s.workflow_as_code_status + s.flow_status, + c.flow_status AS completed_flow_status, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status) IS NOT NULL + AS is_wac, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status)->'_approval' + AS wac_approval, + COALESCE(s.flow_status, c.flow_status)->'approval_conditions' + AS approval_conditions, + NULLIF(COALESCE(f.summary, sc.summary), '') AS flow_summary FROM v2_job j LEFT JOIN v2_job_status s ON s.id = j.id + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN flow f + ON j.kind = 'flow' AND f.workspace_id = j.workspace_id AND f.path = j.runnable_path + LEFT JOIN script sc + ON j.kind = 'script' AND sc.workspace_id = j.workspace_id AND sc.hash = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", ) .bind(&job_id) @@ -4883,31 +5024,31 @@ async fn get_approval_info( .await? .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; - let is_wac = row.workflow_as_code_status.is_some(); + let is_wac = row.is_wac; + let run_ac = row + .approval_conditions + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()); // Extract approval info based on WAC vs classic flow - let (form_schema, description, default_args, enums, approval_conditions, hide_cancel) = + let (form_schema, description, default_args, enums, approval_conditions, hide_cancel, step) = if is_wac { - let approval_meta = row - .workflow_as_code_status - .as_ref() - .and_then(|v| v.get("_approval")); + let approval_meta = row.wac_approval.as_ref(); let form = approval_meta.and_then(|m| m.get("form").cloned()); let default_args = approval_meta.and_then(|m| m.get("default_args").cloned()); let enums = approval_meta.and_then(|m| m.get("enums").cloned()); let description = approval_meta.and_then(|m| m.get("description").cloned()); - let ac = row - .flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()); - (form, description, default_args, enums, ac, None) + let skin = approval_meta + .and_then(|m| m.get("skin")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .unwrap_or_default(); + let step = Some(ApprovalStepView { skin, summary: None }); + (form, description, default_args, enums, run_ac, None, step) } else { let fs = row .flow_status .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()); - let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); // For classic flows, form/description come from the flow definition and step result let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); @@ -4967,6 +5108,28 @@ async fn get_approval_info( .and_then(|s| s.resume_form.as_ref()) .map(|rf| serde_json::json!(rf)); let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); + let completed_fs = row + .completed_flow_status + .as_ref() + .filter(|_| fs.is_none()) + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let approval_module = raw_flow + .as_ref() + .zip(fs.as_ref().or(completed_fs.as_ref())) + .and_then(|(flow, status)| last_reached_approval_step(flow, status)); + let ac = run_ac.or_else(|| { + approval_module + .and_then(|module| module.suspend.as_ref()) + .and_then(approval_conditions_from_settings) + }); + let step = approval_module.map(|module| ApprovalStepView { + skin: module + .suspend + .as_ref() + .and_then(|s| s.skin) + .unwrap_or_default(), + summary: module.summary.clone().filter(|s| !s.trim().is_empty()), + }); // Fetch description, default_args, and enums from the step's completed job result let step_job_id = fs @@ -4990,9 +5153,12 @@ async fn get_approval_info( (None, None, None) }; - (form, desc, default_args, enums, ac, hc) + (form, desc, default_args, enums, ac, hc, step) }; + let skin = step.as_ref().map(|s| s.skin).unwrap_or_default(); + let step_summary = step.and_then(|s| s.summary); + let user_auth_required = approval_conditions .as_ref() .map(|ac| ac.user_auth_required) @@ -5022,6 +5188,9 @@ async fn get_approval_info( can_approve: false, user_auth_required, hide_cancel: None, + skin, + step_summary: None, + flow_summary: None, approvers: vec![], view_token: None, })); @@ -5057,6 +5226,9 @@ async fn get_approval_info( can_approve, user_auth_required, hide_cancel, + skin, + step_summary, + flow_summary: row.flow_summary, approvers, view_token, })) @@ -6472,7 +6644,7 @@ pub async fn run_flow_by_path( Query(run_query): Query, args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let (args, trigger_metadata) = get_args_and_trigger_metadata( + let (args, trigger_metadata) = match get_args_and_trigger_metadata( &db, &authed, RunnableId::from_flow_path(flow_path.to_path()), @@ -6480,7 +6652,15 @@ pub async fn run_flow_by_path( &w_id, args, ) - .await?; + .await? + { + WebhookRun::Run(args, trigger_metadata) => (args, trigger_metadata), + // 200 rather than an error: services drop a webhook that keeps failing, and disabling a + // trigger in Windmill must not cost it its registration. + WebhookRun::TriggerDisabled => { + return Ok((StatusCode::OK, NATIVE_TRIGGER_DISABLED_MSG.to_string())) + } + }; let (uuid, _, _, _) = push_flow_job_by_path_into_queue( authed, @@ -6905,7 +7085,7 @@ pub async fn run_script_by_path( Query(run_query): Query, args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let (args, trigger_metadata) = get_args_and_trigger_metadata( + let (args, trigger_metadata) = match get_args_and_trigger_metadata( &db, &authed, RunnableId::from_script_path(script_path.to_path()), @@ -6913,7 +7093,13 @@ pub async fn run_script_by_path( &w_id, args, ) - .await?; + .await? + { + WebhookRun::Run(args, trigger_metadata) => (args, trigger_metadata), + WebhookRun::TriggerDisabled => { + return Ok((StatusCode::OK, NATIVE_TRIGGER_DISABLED_MSG.to_string())) + } + }; let (uuid, _, _) = push_script_job_by_path_into_queue( authed, @@ -6931,6 +7117,16 @@ pub async fn run_script_by_path( Ok((StatusCode::CREATED, uuid.to_string())) } +/// What a webhook delivery resolved to: the arguments to run with, or nothing to run. +pub enum WebhookRun { + Run(PushArgsOwned, Option), + /// The native trigger this delivery belongs to is disabled. + TriggerDisabled, +} + +const NATIVE_TRIGGER_DISABLED_MSG: &str = + "This trigger is disabled in Windmill, so no job was created"; + #[allow(unused)] pub async fn get_args_and_trigger_metadata( db: &DB, @@ -6939,14 +7135,21 @@ pub async fn get_args_and_trigger_metadata( run_query: &RunJobQuery, w_id: &str, args: RawWebhookArgs, -) -> error::Result<(PushArgsOwned, Option)> { +) -> error::Result { use windmill_common::triggers::TriggerMetadata; // Build trigger metadata if this is a native trigger request #[cfg(feature = "native_trigger")] let (trigger_metadata, native_args) = if let Some(service_name_str) = &run_query.service_name { - use crate::native_triggers::{prepare_native_trigger_args, ServiceName}; + use crate::native_triggers::{ + native_trigger_is_enabled, prepare_native_trigger_args, ServiceName, + }; let service_name = ServiceName::try_from(service_name_str.to_owned())?; + if let Some(external_id) = run_query.trigger_external_id.as_deref() { + if !native_trigger_is_enabled(db, w_id, service_name, external_id).await? { + return Ok(WebhookRun::TriggerDisabled); + } + } let metadata = Some(TriggerMetadata::new( run_query.trigger_external_id.clone(), service_name.as_job_trigger_kind(), @@ -6982,7 +7185,7 @@ pub async fn get_args_and_trigger_metadata( .await? }; - Ok((args, trigger_metadata)) + Ok(WebhookRun::Run(args, trigger_metadata)) } #[derive(Deserialize)] @@ -7377,6 +7580,7 @@ async fn log_job_view( } pub async fn run_wait_result_job_by_path_get( + cross_site: CrossSiteGetGuard, method: hyper::http::Method, authed: ApiAuthed, Extension(user_db): Extension, @@ -7389,6 +7593,7 @@ pub async fn run_wait_result_job_by_path_get( check_license_key_valid().await?; let script_path = script_path.to_path(); + let runnable_id = cross_site.script_runnable(script_path)?; check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; if method == http::Method::HEAD { @@ -7401,12 +7606,7 @@ pub async fn run_wait_result_job_by_path_get( args.body = args::Body::HashMap(payload_as_args); let args = args - .to_args_from_runnable( - &db, - &w_id, - RunnableId::from_script_path(script_path), - run_query.skip_preprocessor, - ) + .to_args_from_runnable(&db, &w_id, runnable_id, run_query.skip_preprocessor) .await?; check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; @@ -7825,6 +8025,7 @@ pub async fn stream_flow_by_version( } pub async fn stream_script_by_path( + cross_site: CrossSiteGetGuard, authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, @@ -7833,12 +8034,13 @@ pub async fn stream_script_by_path( method: hyper::http::Method, args: RawWebhookArgs, ) -> error::Result { + let runnable_id = cross_site.script_runnable(script_path.to_path())?; stream_job( authed, db, user_db, w_id, - RunnableId::from_script_path(script_path.to_path()), + runnable_id, args, run_query, method == http::Method::GET, @@ -11712,6 +11914,7 @@ mod approval_view_gate_tests { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -11806,4 +12009,58 @@ mod approval_view_gate_tests { "trigger@example.com" )); } + + #[test] + fn approval_step_is_the_last_one_passed() { + let flow: FlowValue = serde_json::from_value(serde_json::json!({ "modules": [ + { "id": "a", "value": { "type": "identity" }, "suspend": {} }, + { "id": "b", "value": { "type": "identity" }, "suspend": {} }, + { "id": "c", "value": { "type": "identity" } } + ]})) + .unwrap(); + let step_at = |step: i32, types: [(&str, bool); 3]| { + let mut status = FlowStatus::new(&flow); + status.step = step; + status.modules = ["a", "b", "c"] + .into_iter() + .zip(types) + .map(|(id, (kind, skipped))| { + serde_json::from_value(serde_json::json!({ + "type": kind, "id": id, "job": Uuid::nil(), "count": 1, + "failed_retries": [], "skipped": skipped + })) + .unwrap() + }) + .collect(); + last_reached_approval_step(&flow, &status).map(|module| module.id.clone()) + }; + let waiting = ("WaitingForEvents", false); + let pending = ("WaitingForPriorSteps", false); + let ran = ("Success", false); + let skipped = ("Success", true); + // Awaiting a's approval: b, itself an approval step, already holds `WaitingForEvents`. + assert_eq!(step_at(1, [ran, waiting, pending]).as_deref(), Some("a")); + assert_eq!(step_at(2, [ran, ran, waiting]).as_deref(), Some("b")); + assert_eq!(step_at(3, [ran, skipped, ran]).as_deref(), Some("a")); + assert_eq!(step_at(0, [pending, pending, pending]), None); + } + + #[test] + fn approved_step_stays_gated_by_its_settings() { + let from_settings = |suspend: serde_json::Value| { + approval_conditions_from_settings(&serde_json::from_value(suspend).unwrap()) + }; + let login = from_settings(serde_json::json!({ + "user_auth_required": true, + "user_groups_required": { "type": "static", "value": ["approvers"] } + })); + assert!(!can_view( + &None, + &login, + Some("f/team/flow"), + "trigger@example.com" + )); + assert_eq!(login.unwrap().user_groups_required, ["approvers"]); + assert!(from_settings(serde_json::json!({})).is_none()); + } } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 054af0f2fa..fc2c773703 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -80,6 +80,7 @@ pub mod azure_proxy_ee; mod azure_proxy_oss; mod capture; mod concurrency_groups; +mod csrf; mod db; mod db_health; mod dbt; @@ -378,6 +379,7 @@ async fn inject_agent_authed( token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, }, job_id: None, }); @@ -958,6 +960,15 @@ pub async fn run_server( #[cfg(not(feature = "enterprise"))] Router::new() }) + .nest("/w/{workspace_id}/git_sync", { + #[cfg(feature = "enterprise")] + { + git_sync_oss::workspaced_git_sync_service() + } + + #[cfg(not(feature = "enterprise"))] + Router::new() + }) .nest( "/w/{workspace_id}/resources_u", public_service().layer(cors.clone()), diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index aa2415f664..ea673610e6 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1265,7 +1265,7 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { "type": "string" @@ -1380,7 +1380,7 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { "type": "string" diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 65c8069b85..ce5cb478cf 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -1421,6 +1421,7 @@ mod tests { token_prefix: None, read_only: false, job_id, + credential_expiry: None, } } diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 2998faa7f3..26eb06c8e8 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -128,6 +128,17 @@ struct WorkspaceReassignment { // ---- Preview helpers ---- +/// The principal a departing member's runnables run as, or `None` when none of them are theirs +/// to hand over. `usr.username` is constrained to `[\w-]+`, so a member is `u/{username}`, except +/// a legacy `group-*` username, which canonicalizes to the group it names: what runs under that +/// principal runs as the group, which outlives the member. `None` binds NULL, which the +/// `on_behalf_of = $n` queries then match nowhere. +fn departing_principal(username: &str) -> Option { + use windmill_common::users::{username_to_permissioned_as, PERMISSIONED_AS_GROUP_PREFIX}; + let principal = username_to_permissioned_as(username); + (!principal.starts_with(PERMISSIONED_AS_GROUP_PREFIX)).then_some(principal) +} + async fn get_offboard_preview( db: impl sqlx::PgExecutor<'_> + Copy, w_id: &str, @@ -136,10 +147,8 @@ async fn get_offboard_preview( ) -> Result { let user_prefix = format!("u/{}/%", username); let user_owner = format!("u/{}", username); - // Same form the mutation reassigns, so preview and execution cannot disagree. `usr.username` - // is constrained to `[\w-]+`, so a member is always named `u/{username}` — the address form a - // principal can also take names an account with no `usr` row, which is nobody offboardable. - let departing = windmill_common::users::username_to_permissioned_as(username); + // Same form the mutation reassigns, so preview and execution cannot disagree. + let departing = departing_principal(username); // ---- Owned objects (under u/{username}/) ---- let scripts = sqlx::query_scalar!( @@ -244,17 +253,17 @@ async fn get_offboard_preview( // ---- Operator references (not under user's path) ---- let obo_scripts = sqlx::query_scalar!( "SELECT path FROM script WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived AND NOT deleted", - &departing, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_flows = sqlx::query_scalar!( "SELECT path FROM flow WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived", - &departing, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_apps = sqlx::query_scalar!( "SELECT path FROM app WHERE policy->>'on_behalf_of' = $1 AND NOT path LIKE $2 AND workspace_id = $3", - &user_owner, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_schedules = sqlx::query_scalar!( @@ -831,7 +840,7 @@ async fn offboard_user_from_workspace<'c>( new_permissioned_as: &str, ) -> Result { let new_prefix = reassign_to.to_string(); - let departing = windmill_common::users::username_to_permissioned_as(username); + let departing = departing_principal(username); // The app policy stores an address beside its principal, and script/flow keep one for the // workers that still read it, so the replacement's is resolved here. @@ -871,7 +880,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( "UPDATE script SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -912,7 +921,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( "UPDATE flow SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -925,7 +934,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( r#"UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of'], to_jsonb($1::text)), ARRAY['on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('script', 'flow') AND value->>'on_behalf_of' = $2 AND workspace_id = $3"#, new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -951,9 +960,21 @@ async fn offboard_user_from_workspace<'c>( "UPDATE app SET policy = jsonb_set( jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)), ARRAY['on_behalf_of_email'], to_jsonb($4::text) - ) WHERE policy->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + ) WHERE policy->>'on_behalf_of' = $2 AND workspace_id = $3", &new_permissioned_as, - username, + departing.as_deref(), + w_id, + new_on_behalf_of_user_email + ) + .execute(&mut **tx) + .await?; + + // An app draft carries a copy of the deployed policy and is deployed from it, so it needs + // the same pair rewritten — the draft sweep above only covers scripts and flows. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2 AND workspace_id = $3"#, + new_permissioned_as, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 593ea3b298..204b30010d 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -13,19 +13,25 @@ use sha2::Sha256; use sqlx::types::Uuid; use std::collections::HashMap; use windmill_common::error::{to_anyhow, Error}; +use windmill_common::flows::ApprovalSkin; +use windmill_common::utils::truncate_with_ellipsis; use windmill_common::variables::{get_secret_value_as_admin, get_workspace_key}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; use crate::{ approvals::{ - extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, - MessageFormat, QueryButtonText, QueryDefaultArgsJson, QueryDynamicEnumJson, - QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, + extract_w_id_from_resume_url, get_approval_step_skin, handle_resume_action, + ApprovalFormDetails, FieldType, MessageFormat, QueryButtonText, QueryDefaultArgsJson, + QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, }, auth::OptTokened, }; +// Slack rejects a button value over 2000 characters, and with it the whole post. The button value +// carries the message on to the modal, so the message is shortened to fit. +const SLACK_BUTTON_VALUE_MAX_CHARS: usize = 2000; + #[derive(Deserialize, Debug)] pub struct SlackFormData { payload: String, @@ -127,6 +133,9 @@ struct PrivateMetadata { // HMAC over (w_id, resource_path) keyed on the workspace key; minted when the modal is // built, required by `handle_submission` before the resource_path is decrypted. signature: Option, + // Only selects the wording of the updated channel message, so it is left unsigned. + #[serde(default)] + skin: ApprovalSkin, } // Opportunistic transport-level check: when `SLACK_SIGNING_SECRET` is configured we verify @@ -432,6 +441,7 @@ async fn handle_submission( let container: Container = private_metadata.container; let hide_cancel = private_metadata.hide_cancel; let signature = private_metadata.signature; + let skin = private_metadata.skin; // If hide_cancel is true, we don't need to extract information from the private_metadata if hide_cancel.unwrap_or(false) && action == "cancel" { @@ -463,7 +473,7 @@ async fn handle_submission( tracing::warn!("Failed to resolve slack token for {w_id}/{resource_path}: {e:#}"); Error::BadRequest("Invalid Slack callback request".to_string()) })?; - update_original_slack_message(action, slack_token, container).await?; + update_original_slack_message(action, slack_token, container, skin).await?; Ok(()) } @@ -475,14 +485,19 @@ async fn transform_schemas( required: Option>, default_args_json: Option<&serde_json::Value>, dynamic_enums_json: Option<&serde_json::Value>, + skin: ApprovalSkin, ) -> Result { tracing::debug!("Resume urls: {:#?}", urls); + let link_label = match skin { + ApprovalSkin::Detailed => "Flow suspension details", + ApprovalSkin::Minimal => "View in Windmill", + }; let mut blocks = vec![serde_json::json!({ "type": "section", "text": { "type": "mrkdwn", - "text": format!("{}\n<{}|Flow suspension details>", text, urls.approvalPage), + "text": format!("{}\n<{}|{link_label}>", text, urls.approvalPage), } })]; @@ -918,10 +933,6 @@ async fn send_slack_message( value["approver"] = serde_json::json!(approver); } - if let Some(message) = message { - value["message"] = serde_json::json!(message); - } - if let Some(default_args_json) = default_args_json { value["default_args_json"] = default_args_json.clone(); } @@ -950,33 +961,8 @@ async fn send_slack_message( .map_err(|e| Box::new(e) as Box)?; value["signature"] = serde_json::json!(signature); - let payload = serde_json::json!({ - "channel": channel_id, - "text": "A flow has been suspended. Please approve or reject the flow.", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A flow has been suspended. Please approve or reject the flow." - } - }, - { - "type": "actions", - "elements": [ - { - "type": "button", - "text": { - "type": "plain_text", - "text": "View" - }, - "action_id": "open_modal", - "value": value.to_string() - } - ] - } - ] - }); + let skin = get_approval_step_skin(db, w_id, job_id, flow_step_id).await; + let payload = channel_message_payload(channel_id, skin, message, value); tracing::debug!("Payload: {:?}", payload); @@ -1000,6 +986,88 @@ async fn send_slack_message( Ok(StatusCode::OK) } +/// The channel post announcing the approval. Its button hands `button_value` to the modal, with +/// `message` added, shortened to what Slack's button value limit leaves room for. +fn channel_message_payload( + channel_id: &str, + skin: ApprovalSkin, + message: Option<&str>, + mut button_value: serde_json::Value, +) -> serde_json::Value { + let message = message.map(|m| message_fitting_button_value(&button_value, m)); + if let Some(message) = &message { + button_value["message"] = serde_json::json!(message); + } + let (text, section, button_label) = match skin { + ApprovalSkin::Detailed => { + let text = "A flow has been suspended. Please approve or reject the flow."; + (text, text.to_string(), "View") + } + ApprovalSkin::Minimal => { + let mut section = "*Approval requested*".to_string(); + if let Some(message) = &message { + section.push('\n'); + section.push_str(message); + } + ("Approval requested", section, "Review") + } + }; + + serde_json::json!({ + "channel": channel_id, + "text": text, + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": section + } + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": button_label + }, + "action_id": "open_modal", + "value": button_value.to_string() + } + ] + } + ] + }) +} + +/// The longest prefix of `message` that keeps `button_value` carrying it within Slack's limit. +fn message_fitting_button_value(button_value: &serde_json::Value, message: &str) -> String { + let mut with_message = button_value.clone(); + let mut fits = |max_chars: usize| { + let fitted = truncate_with_ellipsis(message, max_chars); + with_message["message"] = serde_json::json!(fitted); + (with_message.to_string().chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS).then_some(fitted) + }; + if let Some(whole) = fits(usize::MAX) { + return whole; + } + // Searched on the serialized length, which escaping makes longer than the raw prefix, and + // which grows with every character kept. + let (mut shortest, mut longest) = + (0, message.chars().count().min(SLACK_BUTTON_VALUE_MAX_CHARS)); + while shortest < longest { + let mid = (shortest + longest + 1) / 2; + if fits(mid).is_some() { + shortest = mid; + } else { + longest = mid - 1; + } + } + fits(shortest).unwrap_or_else(|| truncate_with_ellipsis(message, 0)) +} + async fn get_modal_blocks( db: DB, w_id: &str, @@ -1034,7 +1102,7 @@ async fn get_modal_blocks( ) .await?; - let ApprovalFormDetails { message_str, urls, schema } = approval_details; + let ApprovalFormDetails { message_str, urls, schema, skin } = approval_details; // Get the card content let card_content = transform_schemas( @@ -1063,6 +1131,7 @@ async fn get_modal_blocks( }), default_args_json, dynamic_enums_json, + skin, ) .await?; @@ -1077,6 +1146,7 @@ async fn get_modal_blocks( resume_button_text, cancel_button_text, &private_metadata_signature, + skin, ))) } @@ -1090,27 +1160,32 @@ fn construct_payload( resume_button_text: Option<&str>, cancel_button_text: Option<&str>, signature: &str, + skin: ApprovalSkin, ) -> serde_json::Value { + let (title, resume_label, cancel_label) = match skin { + ApprovalSkin::Detailed => ("Workflow Suspended", "Resume Workflow", "Cancel Workflow"), + ApprovalSkin::Minimal => ("Approval request", "Approve", "Reject"), + }; let mut view = serde_json::json!({ "type": "modal", "callback_id": "submit_form", "notify_on_close": true, "title": { "type": "plain_text", - "text": "Workflow Suspended" + "text": title }, "blocks": blocks, "submit": { "type": "plain_text", - "text": resume_button_text.unwrap_or("Resume Workflow") + "text": resume_button_text.unwrap_or(resume_label) }, - "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature }).to_string(), + "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature, "skin": skin }).to_string(), }); if !hide_cancel { view["close"] = serde_json::json!({ "type": "plain_text", - "text": cancel_button_text.unwrap_or("Cancel Workflow") + "text": cancel_button_text.unwrap_or(cancel_label) }); } @@ -1193,11 +1268,13 @@ async fn update_original_slack_message( action: &str, token: String, container: Container, + skin: ApprovalSkin, ) -> Result<(), Error> { - let message = if action == "resume" { - "\n\n*Workflow has been resumed!* :white_check_mark:" - } else { - "\n\n*Workflow has been canceled!* :x:" + let message = match (skin, action == "resume") { + (ApprovalSkin::Detailed, true) => "\n\n*Workflow has been resumed!* :white_check_mark:", + (ApprovalSkin::Detailed, false) => "\n\n*Workflow has been canceled!* :x:", + (ApprovalSkin::Minimal, true) => "*Approved* :white_check_mark:", + (ApprovalSkin::Minimal, false) => "*Rejected* :x:", }; let final_blocks = vec![serde_json::json!({ @@ -1242,3 +1319,66 @@ async fn update_original_slack_message( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_message_keeps_the_channel_post_within_slack_limits() { + let button_value = serde_json::json!({ + "w_id": "demo", + "job_id": Uuid::nil(), + "path": "u/admin/slack", + "channel": "C0123456789", + "flow_step_id": "a", + "signature": "f".repeat(64), + }); + let carried = |skin, message: &str| { + let payload = channel_message_payload("C1", skin, Some(message), button_value.clone()); + let button = payload["blocks"][1]["elements"][0]["value"] + .as_str() + .unwrap() + .to_string(); + assert!(button.chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS); + let section = payload["blocks"][0]["text"]["text"].as_str().unwrap(); + assert!(section.chars().count() <= 3000); + serde_json::from_str::(&button) + .unwrap() + .message + .unwrap() + }; + // Quotes and newlines each cost two characters once escaped into the button value. + let message = "Expense \"offsite\" line\n".repeat(1000); + for skin in [ApprovalSkin::Detailed, ApprovalSkin::Minimal] { + let kept = carried(skin, &message); + let kept = kept.strip_suffix("...").unwrap(); + assert!(message.starts_with(kept)); + assert!(kept.chars().count() > 1_000); + assert_eq!(carried(skin, "Short message"), "Short message"); + } + } + + #[test] + fn minimal_skin_survives_the_modal_round_trip() { + let container = Container { message_ts: "1".to_string(), channel_id: "C1".to_string() }; + let payload = construct_payload( + serde_json::json!([]), + false, + "trigger", + "https://example.com/resume", + "u/admin/slack", + container, + None, + None, + "signature", + ApprovalSkin::Minimal, + ); + let view = &payload["view"]; + assert_eq!(view["submit"]["text"], "Approve"); + assert_eq!(view["close"]["text"], "Reject"); + let metadata: PrivateMetadata = + serde_json::from_str(view["private_metadata"].as_str().unwrap()).unwrap(); + assert_eq!(metadata.skin, ApprovalSkin::Minimal); + } +} diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 06a79d022e..22645a2ac4 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -107,6 +107,14 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< .header("Cross-Origin-Resource-Policy", "cross-origin"); } + // Login and its siblings carry a different `rd` on every page that links + // to them, so a crawler meets thousands of URLs for one form. The app is + // client-rendered, so a meta tag only exists after a render pass; the + // header is seen on the first fetch. + if original_path.starts_with("/user/") { + res = res.header("X-Robots-Tag", "noindex, nofollow"); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index e58aab44a2..387ba56589 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -1,6 +1,7 @@ use super::{ - http_trigger_args::RawHttpTriggerArgs, refresh_routers, AuthenticationMethod, HttpMethod, - RequestType, TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, + effective_allowed_origins, http_trigger_args::RawHttpTriggerArgs, match_origin, + refresh_routers, AuthenticationMethod, HttpMethod, RequestType, TriggerRoute, + HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, }; use crate::{ auth::{AuthCache, OptTokened}, @@ -24,6 +25,7 @@ use std::{collections::HashMap, sync::Arc}; use windmill_common::{ db::UserDB, error::{Error, Result}, + global_settings::HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS, jobs::JobTriggerKind, triggers::{TriggerKind, TriggerMetadata}, utils::{not_found_if_none, StripPath}, @@ -37,12 +39,222 @@ use { windmill_object_store::build_object_store_client, }; +/// Which router a request's CORS decision must be looked up in. +/// +/// A preflight names the method it is asking about in +/// `Access-Control-Request-Method`; the routers are keyed by method, so without +/// that header there is nothing to look up. +fn cors_lookup_method(req: &axum::extract::Request) -> Option { + let method = req.method(); + if method == http::Method::OPTIONS { + req.headers() + .get(http::header::ACCESS_CONTROL_REQUEST_METHOD) + .and_then(|method| method.to_str().ok()) + .and_then(|method| http::Method::try_from(method).ok()) + .as_ref() + .and_then(routable_method) + } else { + routable_method(method) + } +} + +/// The router key a request method maps to. `HEAD` resolves the `GET` route it +/// mirrors, and does so for a preflight naming it too: browsers send +/// `Access-Control-Request-Method: HEAD` when the HEAD carries a non-safelisted +/// header, and answering that preflight from a different route than the request +/// itself resolves is how the two come to disagree. +fn routable_method(method: &http::Method) -> Option { + if method == http::Method::HEAD { + Some(HttpMethod::Get) + } else { + HttpMethod::try_from(method).ok() + } +} + +/// The key to look a request up by, matching what `route_job` resolves it to. +/// +/// `Path` percent-decodes before `get_http_route_trigger` builds its +/// lookup key, so decoding here is what keeps the two agreeing: on the raw path, +/// `/us%65rs` misses the trigger registered at `/users` that goes on to serve the +/// request, and the response would carry the permissive default instead of that +/// trigger's allowlist. +fn cors_lookup_path(raw_path: &str) -> Option { + let decoded = urlencoding::decode(raw_path).ok()?; + // `StripPath::to_path` strips one leading slash and the handler trims + // trailing ones, before a single `/` is prefixed back on. + let stripped = decoded.strip_prefix('/').unwrap_or(&decoded); + Some(format!("/{}", stripped.trim_end_matches('/'))) +} + +/// What the middleware should stamp, decided while the routers guard is held. +/// +/// Deliberately small and owned: the allowlist itself never leaves the guard, +/// so a large one is scanned in place instead of being copied per request onto +/// a path an unauthenticated preflight can reach. +#[derive(Clone)] +enum CorsDecision { + /// No allowlist applies, so the permissive default stands. + Unrestricted, + /// An allowlist applies. `allow_origin` is the value to echo, present only + /// when the request's own `Origin` is on the list. + Restricted { route_method: Option, allow_origin: Option }, + /// The routers could not be read, so nothing is known about this path. + Unavailable, +} + +/// Whether a route actually serves a static website, rather than merely saying +/// it does. +/// +/// `is_static_website` is a caller-set flag that validation ties to nothing: a +/// route can carry it while having no assets configured and a `script_path` +/// that `route_job` runs regardless. Keying the exemption off the flag alone +/// would let one boolean disable a route's allowlist and hand its runnable back +/// the `wm_headers` escape hatch, so the assets have to be there too. +fn serves_a_static_website(trigger: &TriggerRoute) -> bool { + trigger.is_static_website && trigger.static_asset_config.is_some() +} + +/// A static website is never subject to an allowlist, its own included. It has +/// no authentication of its own — the editor does not offer any — so it hands +/// out public files that any non-browser client can already fetch, and +/// restricting which browsers may read them protects nothing while breaking the +/// cross-origin uses that do consult CORS: a webfont, a `crossorigin` asset, a +/// `fetch`. +/// +/// A single-file static asset is not exempt. That one can carry an +/// `authentication_method`, so its content need not be public, and an allowlist +/// is what keeps another origin from reading a response its own credentials +/// would not have obtained. +/// +/// The CORS verdict for a request, published by whoever resolved its trigger. +/// +/// The middleware stamps headers after the handler returns, but only the +/// handler knows which trigger it actually served. Re-deriving that from the +/// routers cache is a second lookup which can disagree with the first when a +/// route is edited, deleted or widened mid-request, and every ordering of the +/// two is wrong in some case. So the verdict travels with the request instead +/// of being worked out twice. +#[derive(Clone, Default)] +struct ResolvedCorsPolicy(std::sync::Arc>); + +impl ResolvedCorsPolicy { + /// Record what the trigger being served allows. Called once, where the + /// route is resolved, so the answer cannot drift from the response. + fn publish(&self, trigger: &TriggerRoute, method: Option, headers: &HeaderMap) { + let decision = if serves_a_static_website(trigger) { + CorsDecision::Unrestricted + } else { + let instance_default = HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.load(); + match effective_allowed_origins( + trigger.allowed_origins.as_deref(), + instance_default.as_slice(), + ) { + None => CorsDecision::Unrestricted, + Some(allowed_origins) => CorsDecision::Restricted { + route_method: method, + allow_origin: match_origin(allowed_origins, headers.get(http::header::ORIGIN)), + }, + } + }; + let _ = self.0.set(decision); + } + + fn published(&self) -> Option { + self.0.get().cloned() + } +} + +/// Decide the CORS answer from the routers cache, for a request no handler +/// published a verdict for: a preflight, an unknown path, or any failure ahead +/// of the publish — authentication included, which runs after the route itself +/// resolves. +/// +/// Loads the routers when the cache is cold, the way `get_http_route_trigger` +/// does, so a preflight is answered from the same view of the routes as the +/// request that follows it. +async fn resolve_cors_decision( + db: &DB, + http_method: HttpMethod, + requested_path: &str, + origin: Option<&http::HeaderValue>, +) -> CorsDecision { + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + + let routers_cache = if routers_cache.routers.is_empty() { + drop(routers_cache); + match refresh_routers(db, false).await { + Ok((_, routers_cache)) => routers_cache, + Err(err) => { + tracing::error!("Could not load HTTP routers to resolve CORS: {err:#}"); + return CorsDecision::Unavailable; + } + } + } else { + routers_cache + }; + + let Some(router) = routers_cache.routers.get(&http_method) else { + return CorsDecision::Unavailable; + }; + + let route = router.at(requested_path).ok(); + if route + .as_ref() + .is_some_and(|trigger| serves_a_static_website(trigger.value)) + { + return CorsDecision::Unrestricted; + } + let route_allowed_origins = route + .as_ref() + .and_then(|trigger| trigger.value.allowed_origins.as_deref()); + + let instance_default = HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.load(); + match effective_allowed_origins(route_allowed_origins, instance_default.as_slice()) { + None => CorsDecision::Unrestricted, + Some(allowed_origins) => CorsDecision::Restricted { + route_method: route.map(|_| http_method), + allow_origin: match_origin(allowed_origins, origin), + }, + } +} + async fn conditional_cors_middleware( - req: axum::extract::Request, + Extension(db): Extension, + mut req: axum::extract::Request, next: axum::middleware::Next, ) -> Response { + let origin = req.headers().get(http::header::ORIGIN).cloned(); + // Owned before `next.run` consumes the request. `&Request` is not `Send` + // (`Body` is not `Sync`), so nothing borrowed from it can cross the await. + // The URI is carried rather than the decoded path: cloning it is a refcount + // bump, while decoding allocates, and only the fallback below ever needs it. + let lookup_method = cors_lookup_method(&req); + let uri = req.uri().clone(); + + let resolved = ResolvedCorsPolicy::default(); + req.extensions_mut().insert(resolved.clone()); + let mut response = next.run(req).await; + let decision = match resolved.published() { + // The handler resolved a trigger and said what it served under. That is + // the policy this response was produced with, so nothing else can be + // more authoritative. + Some(decision) => decision, + // No verdict was published: a preflight, an unknown path, or a request + // that failed before reaching the publish, authentication included. No + // runnable produced this body, so reading the cache cannot contradict + // anything. + None => match lookup_method.zip(cors_lookup_path(uri.path())) { + Some((method, path)) => { + resolve_cors_decision(&db, method, &path, origin.as_ref()).await + } + // Not a preflight, not a routable method, or a path that does not + // decode. + None => CorsDecision::Unrestricted, + }, + }; + let headers = response.headers_mut(); // Check existing headers first to determine what not to insert @@ -67,18 +279,65 @@ async fn conditional_cors_middleware( } } - // Insert only the missing headers - if !not_insert_origin { - headers.insert( - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, - http::HeaderValue::from_static("*"), - ); + match &decision { + CorsDecision::Restricted { allow_origin, .. } => { + // A configured allowlist decides, overriding any `wm_headers` value + // the runnable set. The preflight is answered before any code runs, + // so config is the only thing it can consult; letting the response + // widen what the preflight advertised would make the two disagree + // and leave the allowlist bounding nothing. A route escapes a + // stricter instance default — `wm_headers` included — by setting + // its own list to `*`. + match allow_origin { + Some(value) => { + headers.insert(http::header::ACCESS_CONTROL_ALLOW_ORIGIN, value.clone()) + } + // No match: omit the header entirely so the browser blocks the + // read, and drop any value the runnable set. + None => headers.remove(http::header::ACCESS_CONTROL_ALLOW_ORIGIN), + }; + // Appended, not inserted: the answer now depends on the request's + // Origin, and a shared cache that ignores it would hand one + // origin's response to another. + headers.append(http::header::VARY, http::HeaderValue::from_static("origin")); + } + // The routers could not be read, so nothing is known about this path; + // only a preflight or an unresolved request reaches here. Answering a + // preflight permissively would let a disallowed origin go on to invoke + // a runnable whose purpose may be a side effect. + CorsDecision::Unavailable => { + headers.remove(http::header::ACCESS_CONTROL_ALLOW_ORIGIN); + } + CorsDecision::Unrestricted => { + if !not_insert_origin { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, + http::HeaderValue::from_static("*"), + ); + } + } } if !not_insert_methods { + // A route accepts exactly one method, so advertising all seven + // overstates it. Only a route under an allowlist gets the narrower + // answer; an unrestricted one advertises the full supported set, since + // narrowing it would say something about a route the response is not + // otherwise willing to disclose. + let restricted_method = match &decision { + CorsDecision::Restricted { route_method, .. } => *route_method, + _ => None, + }; headers.insert( http::header::ACCESS_CONTROL_ALLOW_METHODS, - http::HeaderValue::from_static("GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"), + http::HeaderValue::from_static(match restricted_method { + Some(HttpMethod::Get) => "GET, OPTIONS", + Some(HttpMethod::Post) => "POST, OPTIONS", + Some(HttpMethod::Put) => "PUT, OPTIONS", + Some(HttpMethod::Delete) => "DELETE, OPTIONS", + Some(HttpMethod::Patch) => "PATCH, OPTIONS", + None => "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS", + }), ); } @@ -237,6 +496,7 @@ async fn route_job( Extension(db): Extension, Extension(user_db): Extension, Extension(auth_cache): Extension>, + Extension(cors_policy): Extension, OptTokened { token }: OptTokened, Path(route_path): Path, headers: HeaderMap, @@ -255,6 +515,10 @@ async fn route_job( .await .map_err(|e| e.into_response())?; + // Publish before anything else can fail: the CORS middleware stamps this + // response either way, and it must reflect the trigger actually served. + cors_policy.publish(&trigger, routable_method(&args.0.metadata.method), &headers); + if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { return Err(Error::NotFound(format!( "Runnable path of HTTP route at path: {}", diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 8804004f2b..4dd073b3af 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -55,6 +55,7 @@ pub fn global_service() -> Router { .route("/rename/{user}", post(rename_user)) .route("/onboarding", post(submit_onboarding_data)) .route("/ext_jwt_tokens", get(list_ext_jwt_tokens)) + .route("/guests", get(list_guests)) .route( "/offboard_preview/{user}", get(crate::offboarding::global_offboard_preview), @@ -141,6 +142,58 @@ async fn list_ext_jwt_tokens( Ok(Json(rows)) } +#[derive(serde::Serialize, sqlx::FromRow)] +pub struct GuestActivity { + pub email: String, + pub workspaces: Vec, + pub first_seen: chrono::NaiveDate, + pub last_seen: chrono::NaiveDate, +} + +#[derive(serde::Serialize)] +pub struct GuestList { + pub usage: windmill_common::workspaces::GuestUsage, + pub guests: Vec, +} + +#[derive(serde::Deserialize)] +struct ListGuestsQuery { + page: Option, + per_page: Option, +} + +/// The distinct guests of the trailing window, the set the allowance is counted on, +/// most recently seen first. +async fn list_guests( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> Result> { + require_super_admin(&db, &authed).await?; + + let (per_page, offset) = windmill_common::utils::paginate(windmill_common::utils::Pagination { + page: query.page, + per_page: query.per_page, + }); + let usage = windmill_common::workspaces::guest_usage(&db).await?; + let guests = sqlx::query_as::<_, GuestActivity>( + "SELECT email, array_agg(DISTINCT workspace_id) AS workspaces, + MIN(day) AS first_seen, MAX(day) AS last_seen + FROM guest_activity + WHERE day > CURRENT_DATE - $3 + GROUP BY email + ORDER BY MAX(day) DESC, email + LIMIT $1 OFFSET $2", + ) + .bind(per_page as i64) + .bind(offset as i64) + .bind(windmill_common::workspaces::GUEST_WINDOW_DAYS) + .fetch_all(&db) + .await?; + + Ok(Json(GuestList { usage, guests })) +} + async fn set_password( Extension(db): Extension, Extension(argon2): Extension>>, @@ -208,6 +261,12 @@ async fn rename_user( ))); } + let old_instance_username = + sqlx::query_scalar!("SELECT username FROM password WHERE email = $1", user_email) + .fetch_optional(&mut *tx) + .await? + .flatten(); + sqlx::query!( "UPDATE password SET username = $1 WHERE email = $2", ru.new_username, @@ -216,6 +275,36 @@ async fn rename_user( .execute(&mut *tx) .await?; + // The per-workspace sweep below only reaches accounts with a `usr` row. A superadmin acting + // outside their workspaces has none, yet an app can name them: their principal is + // `u/{password.username}`, which this rename just moved. Matching on the address as well + // keeps a like-named member of some other workspace out of it. + if let Some(old_username) = old_instance_username.filter(|u| *u != ru.new_username) { + let old_principal = windmill_common::users::username_to_permissioned_as(&old_username); + let new_principal = + windmill_common::users::username_to_permissioned_as(&ru.new_username); + sqlx::query!( + "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)) + WHERE policy->>'on_behalf_of' = $2 AND policy->>'on_behalf_of_email' = $3", + &new_principal, + &old_principal, + user_email + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' = $2 + AND value->'policy'->>'on_behalf_of_email' = $3"#, + &new_principal, + &old_principal, + user_email + ) + .execute(&mut *tx) + .await?; + } + let workspace_usernames = sqlx::query!( "SELECT workspace_id, username FROM usr WHERE email = $1", &user_email @@ -701,6 +790,17 @@ async fn update_username_in_workpsace<'c>( .execute(&mut **tx) .await?; + // An app draft carries a copy of the deployed policy, so the rename must reach it + // there too — same reason as the script/flow draft sweep above. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb('u/' || $1))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( "UPDATE app SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", new_username, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 05cd354cd4..828509519e 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -149,9 +149,7 @@ async fn derive_email( if let Some(hit) = cache.get(permissioned_as) { return Ok(Some(hit.clone())); } - // Uncached: the address goes into an archive a client redeploys from, and the write path - // validates the pair it sends back against an uncached lookup. The memo above still holds - // this to one query per distinct principal per export. + // The memo above holds this to one query per distinct principal per export. let email = windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) .await?; @@ -346,7 +344,7 @@ pub(crate) struct ArchiveQueryParams { default_ts: Option, /// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format settings_version: Option, - /// Opt-in: include `extra_perms` on flow / script / app rows. Default `false` + /// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false` /// so cross-workspace tarball imports do not carry over ACLs referring to /// identities that may not exist in the target workspace. `wmill sync pull` /// passes `true` to surface ACLs in the git-tracked yaml. @@ -365,8 +363,8 @@ pub(crate) struct ArchiveQueryParams { /// pre-existing serialization for folders and groups so /// no customer sees a one-time noisy diff on upgrade. /// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}` -/// or null. New surface for flow / script / app, which -/// never carried ACLs in source before this change. +/// or null. New surface for script / flow / app / variable, +/// which never carried ACLs in source before this change. #[derive(Clone, Copy)] pub enum ExtraPermsBehavior { Drop, @@ -665,7 +663,7 @@ pub(crate) async fn tarball_workspace( check_scopes(&authed, || "variables:read".to_string())?; } - // Opt-in behavior for surfacing per-resource ACLs on flow/app rows. + // Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows. // Folder and group rows have always carried `extra_perms` in source and // continue to do so unconditionally (`KeepEvenEmpty`) so existing // customer git repos see no one-time noisy diff. @@ -1002,8 +1000,7 @@ pub(crate) async fn tarball_workspace( Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) })?); } - let var_str = - &to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap(); + let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) .await?; @@ -1435,9 +1432,11 @@ pub(crate) async fn tarball_workspace( // Native triggers (Nextcloud, Google Drive, GitHub) are never // cloned into a fork — a fork only has one if its owner created // it there, so it's always "fork-only" and keeps its own mode. - // No parent-value substitution applies; we only strip the - // webhook token hash. - let native_ignore_keys = vec!["webhook_token_hash"]; + // No parent-value substitution applies; we strip the webhook + // token hash, and `enabled`, which is operational state a sync + // deliberately does not carry — whether a trigger is paused + // belongs to the workspace it runs in, not to the code. + let native_ignore_keys = vec!["webhook_token_hash", "enabled"]; for trigger in native_triggers { let trigger_str = &to_string_without_metadata( @@ -1587,10 +1586,11 @@ pub(crate) async fn tarball_workspace( .await?; // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility - // Server-owned auto-pull state (the HMAC webhook secret + hook id/error and - // the synced-sha / last-pull status) must never leave the server: keep it out - // of export archives and synced repos, and don't let a re-imported workspace - // inherit another install's hook/sync state. Mirrors the GET-settings redaction. + // Server-owned state (the HMAC webhook secret + hook id/error, the + // synced-sha / last-pull status, and what the credential check observed) + // must never leave the server: keep it out of export archives and synced + // repos, and don't let a re-imported workspace inherit another install's + // hook/sync state. Mirrors the GET-settings redaction. fn redact_git_sync_for_export(git_sync: Option) -> Option { let mut git_sync = git_sync?; if let Some(repos) = git_sync @@ -1611,6 +1611,13 @@ pub(crate) async fn tarball_workspace( auto_pull.remove(field); } } + // What this install observed about its own credential: a token + // id and expiry, and a `checked_at` that moves on its own. + // None of it describes the workspace, and in a git-synced + // `wmill.yaml` it would churn the file for no reason. + if let Some(repo) = repo.as_object_mut() { + repo.remove("credential"); + } } } Some(git_sync) diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index e2c3233183..dcf3465a94 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -109,6 +109,8 @@ pep440_rs.workspace = true systemstat.workspace = true size.workspace = true rsa = { workspace = true, optional = true } +spki = { workspace = true } +pkcs1 = { workspace = true } aes-gcm = { workspace = true, optional = true } semver.workspace = true diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 117b41632a..4d5523524c 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -175,10 +175,12 @@ fn is_write_access(access: Option) -> bool { /// producers). Resource / datatable / volume reads stay explicit-`// on`: /// a config/lookup read cascading is more often surprising than wanted. fn is_auto_trigger_kind(kind: AssetKind) -> bool { - // `Dbt` is deliberately NOT here. dbt is the only thing that can produce a - // warehouse relation (`// materialize` takes DuckLake targets only) and a dbt - // run does not dispatch, so a derived `dbt://` edge could never fire — it - // would draw a cascade arrow into a script nothing can wake. + // `Dbt` is deliberately NOT here. A warehouse relation is usually built by + // the dbt project that reads it, and a dbt run does not dispatch, so deriving + // an edge from every `dbt://` read would draw cascade arrows that mostly never + // fire. The relations a native `// materialize manual dbt://…` script writes + // do wake subscribers, but only through an explicit `// on`, which is where + // the author states that this particular relation has such a producer. matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) } @@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs( out } +/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is +/// its ONLY producer. +/// +/// That is the one shape in which subscribing to a warehouse relation can never +/// be woken: a dbt run records the models it built and does not dispatch +/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that +/// declares `// materialize manual dbt://…` fans out on the ordinary path. +/// +/// `None` covers both "some non-dbt script materializes it" and "nothing +/// produces it yet" — the second is the ordinary deploy-order case, identical to +/// every other asset kind, not a dormant edge. +/// +/// **Give it the workspace pool, not an RLS-scoped transaction.** `script` +/// carries RLS while `asset` does not, so a scoped executor hides producers, and +/// the hidden ones fail in the harmful direction: a native producer the deployer +/// cannot read leaves a dbt-only set behind and refuses a subscription that would +/// have fired. What it discloses in exchange is the path of a dbt script building +/// a relation the caller already named, which the workspace asset graph hands out +/// for every `dbt://` node anyway (the source that script wrote stays gated). +/// Callers must therefore already be scoped to `workspace_id`. +/// +/// `deploying_paths` is excluded from the producer set, and has to be: reading +/// committed rows means the deploying script's own are the version being +/// replaced, so one that just dropped its `// materialize` would still count as a +/// producer and let a now-dormant subscription through. Pass every path this +/// deploy is rewriting — under a rename that is the old path as well as the new +/// one, whose committed write row the transaction is about to remove. Excluding +/// them is free of the opposite error, because a script never wakes its own +/// subscription — the dispatcher skips that as a self-loop. +/// +/// A producer another deploy is committing concurrently is invisible either way, +/// and the outcome depends on which side it is. An uncommitted NATIVE producer +/// leaves a dbt-only set and refuses, with a message the user can retry past. An +/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then +/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger +/// row lands, neither side reports the edge it left dormant. Serializing the two +/// is not worth it: they would have to share a per-relation lock, and the ingest +/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding +/// relation locks first inverts that order into a deadlock across the two +/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md). +pub async fn sole_dbt_producer<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_path: &str, + deploying_paths: &[String], +) -> error::Result> { + use crate::scripts::ScriptLang; + let producers = sqlx::query!( + r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang" + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2 + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND a.usage_path <> ALL($3)"#, + workspace_id, + asset_path, + deploying_paths + ) + .fetch_all(executor) + .await?; + if producers + .iter() + .any(|p| !matches!(p.language, ScriptLang::Dbt)) + { + return Ok(None); + } + Ok(producers.into_iter().next().map(|p| p.path)) +} + +/// The set form of [`sole_dbt_producer`], for asking about many relations at +/// once: every `// on dbt://` edge among `relations` whose producers +/// are all dbt scripts, rendered as `dbt://`. +/// +/// A dbt deploy asks this about the relations it just ingested, because that +/// ingest is what can retroactively leave a subscription accepted earlier — when +/// nothing produced the relation — with dbt as its only producer. +/// +/// Spells the predicate the same way its singular sibling does, per subscriber: +/// the producer set excludes the subscriber's own path (a script never wakes +/// itself) and has to be non-empty (nothing produces it yet is deploy order, not +/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would +/// silence this warning with nothing failing. +/// +/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace +/// pool, caller already scoped to `workspace_id`. +pub async fn dormant_dbt_subscriptions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + relations: &[String], +) -> error::Result> { + if relations.is_empty() { + return Ok(vec![]); + } + let refs = relations + .iter() + .map(|r| format!("dbt://{r}")) + .collect::>(); + Ok(sqlx::query_scalar!( + r#"WITH producer AS ( + SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND 'dbt://' || a.path = ANY($2) + ) + SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!" + FROM script_trigger st + WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset' + AND st.trigger_ref = ANY($2) + AND EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language = 'dbt') + AND NOT EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language <> 'dbt') + ORDER BY 1"#, + workspace_id, + &refs + ) + .fetch_all(executor) + .await?) +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index fa1bc96334..73cdfba35d 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -19,7 +19,7 @@ use crate::{ }; /// Whether `label` denotes a user-created token rather than a system token -/// (`session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`). System-token +/// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`). System-token /// labels are load-bearing — session cleanup, super_admin propagation, expiry /// notifications and username overrides all key off them — so they must not be /// user-editable. `None` (no label) is treated as a user token. @@ -36,6 +36,7 @@ pub fn is_user_token(label: Option<&str>) -> bool { // frontend mirror (`label.toLowerCase().startsWith('ephemeral')`) and // the SQL `lower(label) NOT LIKE 'ephemeral%'` guard. l != "session" + && l != GUEST_SESSION_LABEL && !l.to_lowercase().starts_with("ephemeral") && l != "debugger-token" && !l.starts_with("mcp-oauth-") @@ -56,9 +57,40 @@ pub fn is_server_minted_label(label: &str) -> bool { || label.starts_with("ephemeral-script-end-user-") || label == "ephemeral-script" || label == "session" + || label == GUEST_SESSION_LABEL || label.starts_with("mcp-oauth-") } +/// Label on a guest session (the `guest` app execution mode). This is the *grant*: +/// `AuthCache` will resolve a token carrying it into an identity with no account behind +/// it, which nothing else can do. It must therefore stay unforgeable, which is what +/// listing it in [`is_server_minted_label`] buys — `/users/tokens/create` refuses it. +/// +/// Do not move this test onto the token's scopes. Scopes on a user-minted token are +/// caller-supplied and only ever *narrow* (`app_embed`, `raw_app_sdk`), so a scope +/// that granted non-member access would be free for anyone to declare. +pub const GUEST_SESSION_LABEL: &str = "guest_session"; + +/// Whether `label` marks a guest session. See [`GUEST_SESSION_LABEL`]. +/// +/// Reserved in [`is_user_token`] as well as [`is_server_minted_label`]: the former +/// gates relabelling, and a user token that could be relabelled *into* this +/// namespace would become a guest session with no workspace pin — one that +/// authenticates everywhere. +pub fn is_guest_session_label(label: Option<&str>) -> bool { + label == Some(GUEST_SESSION_LABEL) +} + +/// Whether `path` can be spliced into a scope as one literal resource. The scope +/// grammar reserves three characters: `:` separates the parts, `,` separates +/// resources, `*` is a wildcard. App paths are otherwise free-form (spaces, `@`). A +/// leading `/` is refused too: routes strip it, so the scope would never match. +pub fn is_scope_literal_path(path: &str) -> bool { + !path.is_empty() + && !path.starts_with('/') + && !path.chars().any(|c| matches!(c, ':' | ',' | '*')) +} + /// Whether `label` is the one minted for a browser session at login. [`is_server_minted_label`] /// stops a member minting it directly, but `/users/refresh_token` hands one to any authenticated /// caller, so this attributes a request to the UI without proving it: never gate authority on it. @@ -420,6 +452,42 @@ async fn fetch_authed_from_permissioned_as_inner( w_id: &str, conn: &mut sqlx::PgConnection, ) -> Result { + // The `usr` row is the live binding between a `u/` principal and an address, and it is read + // here anyway for the workspace role. Callers may hand us a cached address, so read it before + // anything is granted: `super_admin` and `email_to_igroup` below are keyed on the address + // while the role is keyed on the principal, and an address that no longer belongs to this + // principal — a username freed and reassigned while its previous holder keeps a privileged + // account — would mix one account's role with another's instance privileges. + let member = match permissioned_as.split_once('/') { + Some(("u", name)) => sqlx::query!( + "SELECT is_admin, operator, email FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id + ) + .fetch_optional(&mut *conn) + .await?, + _ => None, + }; + let resolved_email; + let email = match member.as_ref() { + Some(m) => m.email.as_str(), + // No enabled `usr` row. Resolve as `resolve_username_to_email` does: a disabled member's + // own row still wins over the `password` superadmin fallback, so it can never resolve to + // an unrelated superadmin who shares the username (workspace usernames are only unique per + // workspace). Off the member path, which is why it is worth a query that path skips. + None => match permissioned_as.split_once('/') { + Some(("u", name)) => { + resolved_email = + crate::users::resolve_username_to_email(w_id, name, &mut *conn).await?; + // No live binding at all: the supplied address stands. A cached one is at most one + // notify poll stale; accepted, see `users::get_email_from_permissioned_as`. + resolved_email.as_deref().unwrap_or(email) + } + _ => email, + }, + }; + let is_super_admin = permissioned_as == SUPERADMIN_SYNC_EMAIL || email == SUPERADMIN_SECRET_EMAIL || email == SUPERADMIN_NOTIFICATION_EMAIL @@ -433,22 +501,12 @@ async fn fetch_authed_from_permissioned_as_inner( if prefix == "u" { let (is_admin, is_operator) = if is_super_admin { (true, false) + } else if let Some(m) = member.as_ref() { + (m.is_admin, m.operator) } else { - let r = sqlx::query!( - "SELECT is_admin, operator FROM usr where username = $1 AND \ - workspace_id = $2 AND disabled = false", - name, - &w_id - ) - .fetch_optional(&mut *conn) - .await?; - if let Some(r) = r { - (r.is_admin, r.operator) - } else { - return Err(Error::NotFound(format!( - "user {name} not found in workspace {w_id}" - ))); - } + return Err(Error::NotFound(format!( + "user {name} not found in workspace {w_id}" + ))); }; let groups = get_groups_for_user(w_id, &name, email, &mut *conn).await?; diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index 8fb124a33f..157a61f129 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -8,9 +8,11 @@ //! Two things this module is deliberate about: //! //! * **Asset identity is the physical relation.** A model becomes -//! `dbt:////`: the scheme names dbt, which is the -//! only thing that creates one, but the PATH is the relation and never dbt's -//! own `unique_id`. Two projects meet at a handoff — one materializes a mart, +//! `dbt:////`: the scheme names the namespace dbt +//! made — it is the only thing that DERIVES one, while any other language can +//! declare a write to one (decision 25) — but the PATH is the relation and +//! never dbt's own `unique_id`. Two projects meet at a handoff — one +//! materializes a mart, //! the next declares it a `source` — where `model.a.orders` and //! `source.b.analytics.orders` differ but the relation does not, so keying on //! the node id would leave each project an island; a native script reading the @@ -33,8 +35,8 @@ //! Every `pub` mutator in this module — the manifest ones //! (`replace_dbt_manifest`, `clear_dbt_manifest_version`, //! `clear_dbt_editor_graphs`), -//! the snapshot sweep, and the retry-state ones (`move_dbt_run_state`, -//! `clear_dbt_run_state`, `clear_dbt_run_state_if_path_retired`) — takes the +//! the snapshot sweep, and the script-state ones (`move_dbt_script_state`, +//! `clear_dbt_script_state`, `clear_dbt_script_state_if_path_retired`) — takes the //! workspace and the script to act on as plain arguments and enforces nothing: //! **the caller must already have verified write access to that script**, //! exactly like the sibling `assets::replace_static_asset_usage` each is called @@ -188,6 +190,19 @@ fn graph_digest(ingested: &IngestedManifest, relation_root: &str) -> String { .unwrap_or_default() .as_bytes(), ); + // Only when there are any, so a project that never asked for the analysis + // pass keeps the digest it already has. Hashing an empty section + // unconditionally would change every stored digest at once, and every + // dynamic run would then store a full snapshot until its script is + // redeployed — which reads exactly like the suppression above never working. + if !ingested.column_edges.is_empty() { + h.update(b"\0"); + h.update( + serde_json::to_string(&ingested.column_edges) + .unwrap_or_default() + .as_bytes(), + ); + } format!("{:x}", h.finalize()) } @@ -297,6 +312,14 @@ pub async fn prune_dbt_run_graphs( ) .execute(db) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE job_id <> '00000000-0000-0000-0000-000000000000' + AND ingested_at < now() - make_interval(days => $1)", + RUN_GRAPH_RETENTION_DAYS, + ) + .execute(db) + .await?; // In ONE transaction with the orphan sweep: a restart in the gap leaves graph // rows whose marker is gone, and since the sweep runs only when a marker went, // every later call computes `retired == 0` and skips them for good. @@ -325,7 +348,7 @@ pub async fn prune_dbt_run_graphs( // partial index here — all of them `WHERE job_id <> DEPLOYED` — and past the // keep-count is rare, so the ordinary run should pay for neither. if retired > 0 { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} t WHERE t.workspace_id = $1 AND t.script_path = $2 @@ -381,6 +404,18 @@ pub struct IngestedNode { pub severity: Option, pub attached_node: Option, pub columns: Option, + /// The node's real columns, typed and ordered — `[{"name": …, "type": …}]`, + /// from the engine's static analysis. `None` when the project did not ask + /// for it or the engine wrote none. Beside `columns` rather than merged into + /// it: that one is what the author DECLARED, and stays that. + /// + /// Skipped when absent, unlike its neighbours, because `graph_digest` + /// serializes these nodes: emitting `"column_schema":null` would change + /// every stored digest at once, and every dynamic run of a project that + /// never asked for the pass would store a full snapshot until its script is + /// redeployed. + #[serde(skip_serializing_if = "Option::is_none")] + pub column_schema: Option, pub freshness: Option, /// The transform itself, for the graph to render. The copy taken at /// deploy: the file itself is in the script's module bundle. @@ -388,6 +423,40 @@ pub struct IngestedNode { pub original_file_path: Option, } +/// One column-to-column edge of the ingested graph. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +#[serde(default)] +pub struct IngestedColumnEdge { + pub parent_unique_id: String, + pub parent_column: String, + pub child_unique_id: String, + pub child_column: String, + /// dbt's own word: `copy`, `mod` or `scan`. Kept verbatim — the engine's own + /// reader maps those three and passes anything else through, so the set is + /// open. + pub lineage_kind: String, +} + +/// One column of a node, as the engine's static analysis resolved it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexedColumn { + pub name: String, + /// The declared type where `schema.yml` gives one, else the inferred one. + /// Empty when neither is known; the column still belongs to the relation, so + /// only the type is left out. + pub column_type: String, + /// Position in the relation, which is the order the panel lists them in. + pub index: i64, +} + +/// What one `--write-index` pass produced: the column edges of the whole +/// project and the real column schema per node. +#[derive(Debug, Default)] +pub struct ColumnIndex { + pub edges: Vec, + pub columns: HashMap>, +} + // Serde: an agent worker cannot write these tables directly, so it posts the // whole manifest to the server, which stores it with the same function the SQL // path uses. @@ -398,6 +467,9 @@ pub struct IngestedNode { pub struct IngestedManifest { pub nodes: Vec, pub edges: Vec<(String, String)>, + /// Column-to-column lineage, when the project asked for it and the engine + /// produced it. Empty is the normal case — see `attach_column_index`. + pub column_edges: Vec, /// The `asset` rows the owning script produces (models) and consumes /// (sources) — what the lineage graph is drawn from. pub assets: Vec, @@ -405,6 +477,88 @@ pub struct IngestedManifest { pub adapter_type: String, } +/// The most column edges one graph stores. +/// +/// A `scan` edge — the column was read to produce the row, not the value — is +/// emitted from every join key and every predicate column to every output +/// column, so one wide model over a multi-column join contributes columns times +/// predicates edges on its own. The cap is what keeps a project shaped like that +/// from turning one deploy into a multi-million-row insert; past it the lineage +/// is truncated and the rest of the graph is unaffected. +pub const MAX_COLUMN_EDGES: usize = 200_000; + +/// Whether the value travelled along this edge, as opposed to the column merely +/// being read to produce the row. +/// +/// A `scan` edge reaches every output column of its model, so it is most of what +/// a wide project's index holds and the first thing `MAX_COLUMN_EDGES` gives up. +/// It is still stored, for a view that wants indirect influence. +pub fn is_direct(lineage_kind: &str) -> bool { + matches!(lineage_kind, "copy" | "mod") +} + +impl IngestedManifest { + /// Fold one `--write-index` pass into the graph. + /// + /// Both halves are scoped to the nodes this graph already kept: the index + /// describes the whole project, while the graph describes what this script's + /// selection builds plus the parents anchoring its edges, and an edge whose + /// endpoint is absent has nothing to draw. + pub fn attach_column_index(&mut self, index: ColumnIndex) { + let kept: std::collections::HashSet<&str> = + self.nodes.iter().map(|n| n.unique_id.as_str()).collect(); + let mut edges: Vec = index + .edges + .into_iter() + .filter(|e| { + kept.contains(e.parent_unique_id.as_str()) + && kept.contains(e.child_unique_id.as_str()) + }) + .collect(); + // Sorted and deduplicated for the digest, which decides whether a run + // stores a snapshot at all: parquet row order is the engine's and two + // passes over one project must not read as two different graphs. + // + // Direct kinds first, so what the truncation below gives up is `scan` — + // the bulk of a wide project's lineage, and the kind that says the column + // was read to produce the row rather than the value. The + // worker's reader already applies this order while decoding, because the + // memory bound has to; repeating it here is what makes the ordering a + // property of the manifest rather than of one caller's reader, and it is + // the only ordering an index assembled some other way would get. + edges.sort_by(|a, b| { + is_direct(&b.lineage_kind) + .cmp(&is_direct(&a.lineage_kind)) + .then_with(|| a.cmp(b)) + }); + edges.dedup(); + edges.truncate(MAX_COLUMN_EDGES); + self.column_edges = edges; + + let mut columns = index.columns; + for node in self.nodes.iter_mut() { + let Some(mut cols) = columns.remove(&node.unique_id) else { + continue; + }; + if cols.is_empty() { + continue; + } + cols.sort_by_key(|c| c.index); + node.column_schema = Some(serde_json::Value::Array( + cols.into_iter() + // A column the analysis typed as nothing still belongs in + // the list — that it exists is the half `manifest.json` + // could not answer. + .map(|c| match c.column_type.is_empty() { + true => serde_json::json!({ "name": c.name }), + false => serde_json::json!({ "name": c.name, "type": c.column_type }), + }) + .collect(), + )); + } + } +} + /// dbt's `materialized` mapped onto Windmill's write strategy. /// /// The mapping is exact for the four strategies Windmill has, and deliberately @@ -655,6 +809,9 @@ pub fn ingest_manifest( .map(|(k, v)| (k.clone(), v.description.clone().unwrap_or_default())) .collect::>()) }), + // Filled by `attach_column_index` when the project asked for it: + // the manifest carries declared columns only. + column_schema: None, freshness: node.freshness.clone(), // The transform the graph renders. Capped: a project can hold // thousands of models and this is duplicated per deploy, so a @@ -826,6 +983,16 @@ pub async fn replace_dbt_manifest( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2 + AND script_hash = $3 AND job_id = $4", + workspace_id, + script_path, + script_hash, + job_id + ) + .execute(&mut **tx) + .await?; // The marker, before the rows: a graph with no nodes at all is a legitimate // answer for a dynamic run that disabled every model, and the reader must be // able to tell it from a run that stored nothing. @@ -881,7 +1048,7 @@ async fn insert_graph_rows( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, \ resource_type, name, asset_path, materialized, materialize_strategy, unique_key, \ tags, description, test_kind, test_column, test_args, severity, attached_node, \ - columns, freshness, raw_code, original_file_path) ", + columns, column_schema, freshness, raw_code, original_file_path) ", ); q.push_values(chunk, |mut b, n| { b.push_bind(workspace_id) @@ -903,6 +1070,7 @@ async fn insert_graph_rows( .push_bind(&n.severity) .push_bind(&n.attached_node) .push_bind(&n.columns) + .push_bind(&n.column_schema) .push_bind(&n.freshness) .push_bind(&n.raw_code) .push_bind(&n.original_file_path); @@ -926,6 +1094,26 @@ async fn insert_graph_rows( q.push(" ON CONFLICT DO NOTHING"); q.build().execute(&mut **tx).await?; } + + for chunk in ingested.column_edges.chunks(COLUMN_EDGE_INSERT_CHUNK) { + let mut q = sqlx::QueryBuilder::new( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, \ + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind) ", + ); + q.push_values(chunk, |mut b, e| { + b.push_bind(workspace_id) + .push_bind(script_path) + .push_bind(script_hash) + .push_bind(job_id) + .push_bind(&e.parent_unique_id) + .push_bind(&e.parent_column) + .push_bind(&e.child_unique_id) + .push_bind(&e.child_column) + .push_bind(&e.lineage_kind); + }); + q.push(" ON CONFLICT DO NOTHING"); + q.build().execute(&mut **tx).await?; + } Ok(()) } @@ -966,7 +1154,7 @@ pub async fn replace_dbt_editor_graph( ) -> Result<()> { // By job alone, so re-executing one — a zombie recovered onto another // worker — replaces its rows rather than colliding with them. - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = $2 AND script_hash IS NULL" )) @@ -1016,7 +1204,7 @@ pub async fn replace_dbt_editor_graph( .fetch_all(&mut **tx) .await?; if !retired.is_empty() { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = ANY($2) \ AND script_hash IS NULL" @@ -1035,6 +1223,8 @@ pub async fn replace_dbt_editor_graph( const NODE_INSERT_CHUNK: usize = 2000; /// Six columns, so the same ceiling allows far more. const EDGE_INSERT_CHUNK: usize = 8000; +/// Nine columns, and by far the most numerous rows of the three. +const COLUMN_EDGE_INSERT_CHUNK: usize = 6000; /// Clear one VERSION's graph: the delete-by-hash route, which only soft-deletes /// its `script` row and so fires no cascade, and the ingest that finds no @@ -1070,6 +1260,15 @@ pub async fn clear_dbt_manifest_version( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + workspace_id, + script_path, + script_hash + ) + .execute(&mut **tx) + .await?; // The marker too, and every job's: a marker left standing for rows that are // gone is read as a snapshot, and its digest still answers the suppression // check — so an identical run would write nothing and then render an empty @@ -1105,7 +1304,7 @@ pub async fn clear_dbt_editor_graphs( workspace_id: &str, script_path: &str, ) -> Result<()> { - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL" @@ -1118,22 +1317,27 @@ pub async fn clear_dbt_editor_graphs( Ok(()) } -/// Move a dbt script's saved retry state to its new path. +/// Move a dbt script's saved state to its new path: the run `dbt retry` resumes, +/// and the state each environment's deferrals resolve through. /// -/// Keyed by path like the sidecar, but unlike the sidecar it is not -/// regenerated by anything: the deploy re-ingests a manifest, while these are -/// the results of a run that already happened. Clearing on rename would throw -/// away a resumable failure for a cosmetic change, so it travels instead. +/// Keyed by path like the sidecar, but unlike the sidecar neither is regenerated +/// by anything: the deploy re-ingests a manifest, while these are the results of +/// runs that already happened. Clearing on rename would throw away a resumable +/// failure, and every deferral until the next full run, for a cosmetic change — +/// so they travel instead. An artifact too large for its row is unaffected: its +/// key is that publication's own, and the moved row is what names it. /// /// See the mutator contract above: this authorizes nothing. -pub async fn move_dbt_run_state( +pub async fn move_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, old_path: &str, new_path: &str, ) -> Result<()> { // The destination may already hold state from a script that lived there - // before; the incoming row is the newer truth for this project. + // before; the incoming row is the newer truth for this project. What the + // displaced row named in object storage is left there, as a cleared one's is + // — see `clear_dbt_script_state`. sqlx::query!( "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2", workspace_id, @@ -1149,22 +1353,38 @@ pub async fn move_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + new_path + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE dbt_environment_state SET script_path = $3 + WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + old_path, + new_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop the saved retry state, but only once NO live version of the path is -/// left. +/// Drop the saved state, but only once NO live version of the path is left. /// -/// `dbt_run_state`'s key is the path and the principal — one saved run per script -/// per identity it executes as, not -/// one per version — so archiving or deleting a single version must not take it -/// with them: the live version's `dbt retry` would be refused and the -/// partial-failure resume lost. It does not need to be version-scoped either, -/// because `identity` already refuses a resume whose project, warehouse or -/// engine moved. +/// Neither table is keyed by version — `dbt_run_state` by path and principal, +/// `dbt_environment_state` by path and environment — so archiving or deleting a +/// single version must not take them with it: the live version's `dbt retry` +/// would be refused, its partial-failure resume lost, and every deferral would +/// have to wait for another full run to republish. Neither needs to be +/// version-scoped either: `identity` already refuses a resume whose project, +/// warehouse or engine moved, and a deferral resolves relation names, which a +/// new version of the same project spells the same way. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state_if_path_retired( +pub async fn clear_dbt_script_state_if_path_retired( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1179,17 +1399,34 @@ pub async fn clear_dbt_run_state_if_path_retired( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2 + AND NOT EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false)", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop a dbt script's saved retry state. +/// Drop a dbt script's saved state, both halves. /// -/// Archive and delete: `run_results` is not small, the invocation arguments it -/// carries are the user's, and a script later created at the same path would -/// otherwise inherit a stranger's resumable failure. +/// Archive and delete: neither is small, the invocation arguments and manifest +/// they carry are the user's, and a script later created at the same path would +/// otherwise inherit a stranger's resumable failure and defer to a project it +/// has nothing to do with. +/// +/// An artifact too large for its row lives in the instance's object storage, and +/// this leaves it there — as a deleted script leaves its bundle. Reaching it from +/// here would mean an object-store client in this crate and a delete that has to +/// land after the caller's transaction commits, for one object per environment of +/// a script that is gone. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state( +pub async fn clear_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1201,6 +1438,13 @@ pub async fn clear_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } @@ -1765,6 +2009,69 @@ mod tests { assert_eq!(back.assets.len(), ingested.assets.len()); assert_eq!(back.assets[0].path, ingested.assets[0].path); } + + // The index describes the whole PROJECT while the graph describes what this + // script's selection builds, so an edge whose endpoint the graph does not + // hold has nothing to draw and must not be stored. + #[test] + fn column_lineage_is_scoped_to_the_nodes_the_graph_kept() { + let mut i = ingested(); + let kept = "model.jaffle_shop.customers"; + let dropped = "model.other_project.elsewhere"; + i.attach_column_index(ColumnIndex { + edges: vec![ + edge("model.jaffle_shop.orders_daily", "id", kept, "id", "copy"), + edge(dropped, "id", kept, "id", "copy"), + edge(kept, "id", dropped, "id", "copy"), + ], + columns: [ + ( + kept.to_string(), + vec![ + col("total", "Float64", 1), + col("id", "Int32", 0), + col("untyped", "", 2), + ], + ), + (dropped.to_string(), vec![col("id", "Int32", 0)]), + ] + .into(), + }); + assert_eq!( + i.column_edges + .iter() + .map(|e| (e.parent_unique_id.as_str(), e.child_unique_id.as_str())) + .collect::>(), + vec![("model.jaffle_shop.orders_daily", kept)] + ); + // In `column_index` order, and a column the analysis could not type still + // belongs to the relation. + assert_eq!( + node(&i, kept).column_schema, + Some(serde_json::json!([ + {"name": "id", "type": "Int32"}, + {"name": "total", "type": "Float64"}, + {"name": "untyped"}, + ])) + ); + assert!(node(&i, "model.jaffle_shop.orders_daily") + .column_schema + .is_none()); + } + + fn edge(from: &str, from_col: &str, to: &str, to_col: &str, kind: &str) -> IngestedColumnEdge { + IngestedColumnEdge { + parent_unique_id: from.into(), + parent_column: from_col.into(), + child_unique_id: to.into(), + child_column: to_col.into(), + lineage_kind: kind.into(), + } + } + + fn col(name: &str, column_type: &str, index: i64) -> IndexedColumn { + IndexedColumn { name: name.into(), column_type: column_type.into(), index } + } } /// Record one model's state for THIS RUN. diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 0d5ebdc2c3..da9f5f3df2 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -272,7 +272,10 @@ fn format_db_error(message: &str, detail: Option<&str>, hint: Option<&str>) -> S msg } -fn error_source_chain(e: &dyn std::error::Error) -> String { +/// `e` followed by each of its sources, `: `-separated. The result is meant for +/// users, and a `reqwest::Error` renders its request URL: never pass one built +/// from a URL carrying credentials in its userinfo. +pub fn error_source_chain(e: &dyn std::error::Error) -> String { let mut msg = e.to_string(); let mut source = e.source(); while let Some(cause) = source { diff --git a/backend/windmill-common/src/folders.rs b/backend/windmill-common/src/folders.rs index 2a6ba935b6..fa4dfdf3b7 100644 --- a/backend/windmill-common/src/folders.rs +++ b/backend/windmill-common/src/folders.rs @@ -78,8 +78,6 @@ pub async fn resolve_folder_default_on_behalf_of( else { return Ok(None); }; - // Uncached: this pair is written straight onto the runnable, where a stale address would - // contradict the principal it is stored beside. let email = crate::users::get_email_from_permissioned_as_uncached(&permissioned_as, w_id, db).await?; Ok(Some((email, permissioned_as))) diff --git a/backend/windmill-common/src/git_sync_oss.rs b/backend/windmill-common/src/git_sync_oss.rs index 5f8ba771fb..4c7de4fa7b 100644 --- a/backend/windmill-common/src/git_sync_oss.rs +++ b/backend/windmill-common/src/git_sync_oss.rs @@ -1,11 +1,16 @@ #[cfg(feature = "private")] #[allow(unused)] pub use crate::git_sync_ee::*; -#[cfg(not(feature = "private"))] +#[cfg(not(all(feature = "private", feature = "enterprise")))] use sqlx::{Pool, Postgres}; use url::Url; -#[cfg(not(feature = "private"))] +/// Gated on the pair to match [`with_stored_credential`] below, whose callers +/// reach it through this facade un-gated and so depend on it. Nothing routes +/// here today (the one caller imports the enterprise item directly), so this is +/// for uniformity: the next plain caller would otherwise find no definition +/// under `private` without `enterprise`. +#[cfg(not(all(feature = "private", feature = "enterprise")))] pub async fn get_github_app_token_internal( _db: &Pool, _job_token: &str, @@ -15,6 +20,21 @@ pub async fn get_github_app_token_internal( )); } +/// Server-held git credentials are an enterprise feature, so on this build a +/// repository URL authenticates with whatever it already carries. +/// +/// Gated on the pair rather than on `private` alone: `private` does not imply +/// `enterprise`, and the callers are plain (no `#[cfg]`), so a build with one +/// and not the other would find neither this nor the enterprise definition. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn with_stored_credential( + _db: &Pool, + _w_id: &str, + url: String, +) -> crate::error::Result { + Ok(url) +} + lazy_static::lazy_static! { /// Matches a `user:password@` (or `user@`) userinfo component right after the URL scheme. static ref GIT_URL_USERINFO_RE: regex::Regex = diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 5c4b4bddab..c0ed63cd53 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -65,6 +65,10 @@ pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics"; pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics"; pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; +/// Superadmin switch over guest sessions for the whole instance, above the per-workspace +/// one. Read from the table, uncached, by the same gates that read the workspace switch; +/// the superadmin Guests list writes it through `/settings/global/{key}` by this name. +pub const GUEST_ACCESS_DISABLED_SETTING: &str = "guest_access_disabled"; pub const JOB_ISOLATION_SETTING: &str = "job_isolation"; pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; pub const NSJAIL_TMP_BACKING_SETTING: &str = "nsjail_tmp_backing"; @@ -94,6 +98,9 @@ pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation pub const DISABLE_WORKSPACE_INVITE_EMAILS_SETTING: &str = "disable_workspace_invite_emails"; pub const DISABLE_PASSWORD_LOGIN_SETTING: &str = "disable_password_login"; pub const AUTO_LOGIN_PROVIDER_SETTING: &str = "auto_login_provider"; +/// Name of the SAML attribute or OIDC userinfo claim carrying the user's IdP groups. Unset or +/// empty leaves instance-group membership entirely to SCIM. +pub const SSO_GROUPS_CLAIM_SETTING: &str = "sso_groups_claim"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const DISABLE_HUB_SETTING: &str = "disable_hub"; @@ -111,6 +118,7 @@ pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; pub const OTEL_TRACES_RETENTION_SECS_SETTING: &str = "otel_traces_retention_secs"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const HTTP_ROUTE_WORKSPACED_ROUTE_SETTING: &str = "http_route_workspaced_route"; +pub const HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING: &str = "http_route_default_allowed_origins"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version"; pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; @@ -118,6 +126,112 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Instance-wide announcement rendered above every page of the app (maintenance +/// windows, incidents). Readable by any authenticated user, unlike most settings: +/// the banner exists to be shown to everyone, so it must never hold anything the +/// whole instance may not see. +pub const INSTANCE_BANNER_SETTING: &str = "instance_banner"; + +/// Ceiling on the banner message. The banner is a one-or-two-line strip above every +/// page, so anything longer is a layout accident rather than an announcement. +pub const INSTANCE_BANNER_MESSAGE_MAX_LEN: usize = 500; + +/// Ceiling on the banner's link label, which renders as a button inside that same strip. +pub const INSTANCE_BANNER_LINK_LABEL_MAX_LEN: usize = 60; + +/// Validate an [`INSTANCE_BANNER_SETTING`] value. +/// +/// The banner is the one setting rendered to every user of the instance, so its +/// shape is checked at the boundary rather than trusted from the writer: a value +/// that reaches the browser malformed breaks the layout for everyone at once. +/// +/// The link is restricted to http(s) so a stored `javascript:`/`data:` URL can +/// never become the href of an anchor every user sees. +/// +/// Only shapes that would *misrender* are rejected. An enabled banner with no message +/// is left alone deliberately: it renders as nothing, and every write path here runs +/// under the bulk settings save, so rejecting it would fail an admin's whole settings +/// edit — retention, SMTP and all — over a half-typed announcement. +pub fn validate_instance_banner(value: &serde_json::Value) -> Result<(), String> { + let obj = value + .as_object() + .ok_or_else(|| "must be a JSON object".to_string())?; + + // Field types are checked before their contents. Every read below is an `as_str`/ + // `as_bool`, which reports a wrong-typed field as absent — so without this a + // `"link": 123` would skip the URL checks entirely and be stored, and the settings + // form would then throw on it (`link.trim()` on a number) instead of rendering. + for (field, expected, ok) in [ + ( + "enabled", + "a boolean", + obj.get("enabled").is_none_or(|v| v.is_boolean()), + ), + ( + "dismissible", + "a boolean", + obj.get("dismissible").is_none_or(|v| v.is_boolean()), + ), + ( + "message", + "a string", + obj.get("message").is_none_or(|v| v.is_string()), + ), + ( + "severity", + "a string", + obj.get("severity").is_none_or(|v| v.is_string()), + ), + ( + "link", + "a string", + obj.get("link").is_none_or(|v| v.is_string()), + ), + ( + "link_label", + "a string", + obj.get("link_label").is_none_or(|v| v.is_string()), + ), + ] { + if !ok { + return Err(format!("{field} must be {expected}")); + } + } + + for (field, max) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN), + ] { + let len = obj + .get(field) + .and_then(|v| v.as_str()) + .map_or(0, |s| s.chars().count()); + if len > max { + return Err(format!("{field} must be at most {max} characters")); + } + } + + if let Some(severity) = obj.get("severity").and_then(|v| v.as_str()) { + if !matches!(severity, "info" | "warning" | "error") { + return Err("severity must be one of info, warning, error".to_string()); + } + } + + if let Some(link) = obj.get("link").and_then(|v| v.as_str()) { + if !link.trim().is_empty() { + let url = url::Url::parse(link.trim()) + .map_err(|e| format!("link must be an absolute http(s) URL: {e}"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err("link must use the http or https scheme".to_string()); + } + if !url.has_host() { + return Err("link must include a host".to_string()); + } + } + } + + Ok(()) +} /// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. /// @@ -249,6 +363,125 @@ use std::sync::atomic::AtomicBool; lazy_static::lazy_static! { pub static ref HTTP_ROUTE_WORKSPACED_ROUTE: AtomicBool = AtomicBool::new(false); pub static ref DISABLE_PASSWORD_LOGIN: AtomicBool = AtomicBool::new(false); + /// Origins HTTP routes allow cross-origin when they configure none of their + /// own. Empty means unset, which keeps the historical `*`. + pub static ref HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS: arc_swap::ArcSwap> = + arc_swap::ArcSwap::from_pointee(vec![]); +} + +/// Whether an allowlist places no restriction at all. +/// +/// `*` is the explicit "open on purpose" entry, and a route carrying it behaves +/// exactly as an unconfigured one: it is how a route opts out of a stricter +/// instance default, including back into the `wm_headers` escape hatch. +pub fn allows_any_origin(allowed_origins: &[String]) -> bool { + allowed_origins.iter().any(|allowed| allowed == "*") +} + +/// An allowlist is scanned on every request to a restricted route, including +/// the unauthenticated preflight, so its size is a request cost anyone can +/// trigger. +pub const MAX_ALLOWED_ORIGINS: usize = 100; +pub const MAX_ALLOWED_ORIGIN_LEN: usize = 256; + +/// Reject allowlist entries that cannot be compared, stored, or safely allowed. +/// +/// The stored string is only ever an operand: `match_origin` echoes the +/// request's own `Origin` back, never this value, so a malformed entry matches +/// nothing and fails closed. Shapes that merely cannot match are the editor's +/// business to warn about, not this function's to refuse. What is left are the +/// three cases where permissiveness costs something: `null` is what every +/// sandboxed iframe sends, so allowing it would admit any page that can open +/// one; a comma cannot survive the editor's comma-separated field, which would +/// silently split one entry into two and widen the list; and an unbounded list +/// makes every preflight pay for it. +pub fn validate_allowed_origins(allowed_origins: &[String]) -> crate::error::Result<()> { + if allowed_origins.len() > MAX_ALLOWED_ORIGINS { + return Err(crate::error::Error::BadRequest(format!( + "At most {} allowed origins, got {}.", + MAX_ALLOWED_ORIGINS, + allowed_origins.len() + ))); + } + + for origin in allowed_origins { + if origin == "*" { + continue; + } + + let invalid = |reason: &str| { + crate::error::Error::BadRequest(format!( + "Invalid allowed origin '{}': {}.", + origin, reason + )) + }; + + if origin.is_empty() { + return Err(invalid("must not be empty")); + } + if origin.len() > MAX_ALLOWED_ORIGIN_LEN { + return Err(invalid("is longer than any origin a browser sends")); + } + // The editor edits the whole list as one comma-separated field, so an + // entry carrying a comma comes back as two and widens the list. + if origin.contains(',') { + return Err(invalid("must not contain a comma, which separates entries")); + } + if origin.eq_ignore_ascii_case("null") { + return Err(invalid( + "'null' is what a sandboxed iframe sends, so allowing it would allow any page that can open one", + )); + } + // An Origin header is always visible ASCII, so a value outside it can + // never be the string this is compared against. + if !origin.chars().all(|c| c.is_ascii_graphic()) { + return Err(invalid( + "must contain only visible ASCII, with no whitespace", + )); + } + } + + Ok(()) +} + +/// Read [`HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING`] from its stored value. +/// +/// Accepts the comma-separated string the settings UI writes, or a JSON array +/// for anything setting it through the API directly. +pub fn parse_allowed_origins_setting( + value: Option<&serde_json::Value>, +) -> crate::error::Result> { + let origins = match value { + None | Some(serde_json::Value::Null) => vec![], + Some(serde_json::Value::String(raw)) => raw + .split(',') + .map(|origin| origin.trim().to_string()) + .filter(|origin| !origin.is_empty()) + .collect(), + Some(serde_json::Value::Array(entries)) => entries + .iter() + .map(|entry| match entry { + serde_json::Value::String(origin) => Ok(origin.trim().to_string()), + _ => Err(crate::error::Error::BadRequest(format!( + "{} entries must be strings", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ))), + }) + // Not filtered for empties, unlike the string form: there a + // trailing separator naturally yields an empty token, whereas an + // empty array entry is something the caller wrote and validation + // should reject rather than silently drop. + .collect::>>()?, + Some(_) => { + return Err(crate::error::Error::BadRequest(format!( + "{} expected to be a comma-separated string or an array of strings", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ))) + } + }; + + validate_allowed_origins(&origins)?; + Ok(origins) } pub const ENV_SETTINGS: &[&str] = &[ @@ -320,6 +553,30 @@ pub const ENV_SETTINGS: &[&str] = &[ "OTEL_METRICS", "OTEL_TRACING", "OTEL_LOGS", + // The OTEL_EXPORTER_OTLP_*HEADERS variables are left out: they carry exporter API keys, and + // this list is logged at startup and returned to superadmins by `get_local_settings`. + "OTEL_METRICS_ENABLED", + "OTEL_TRACING_ENABLED", + "OTEL_LOGS_ENABLED", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_COMPRESSION", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "OTEL_METRIC_EXPORT_INTERVAL", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_ENVIRONMENT", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_JOB_LOGS", + "OTEL_TRACES_RETENTION_SECS", "DISABLE_S3_STORE", "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", @@ -583,6 +840,61 @@ mod tests { } } + #[test] + fn instance_banner_rejects_unsafe_and_malformed_values() { + // The link becomes the href of an anchor shown to every user of the instance, + // so a non-http(s) scheme must not survive a write. + for link in [ + "javascript:alert(1)", + "data:text/html,", + "vbscript:msgbox(1)", + "not-a-url", + "https://", + ] { + let banner = serde_json::json!({ "enabled": true, "message": "down", "link": link }); + assert!( + validate_instance_banner(&banner).is_err(), + "link '{link}' should be rejected" + ); + } + // A wrong-typed field reads as absent to every accessor here, so without an + // explicit type check it would skip validation and be stored. + for bad in [ + serde_json::json!({ "enabled": true, "message": "down", "link": 123 }), + serde_json::json!({ "enabled": true, "message": "down", "link_label": ["a"] }), + serde_json::json!({ "enabled": true, "message": { "text": "down" } }), + serde_json::json!({ "enabled": true, "message": "down", "severity": 2 }), + serde_json::json!({ "enabled": "yes", "message": "down" }), + serde_json::json!({ "enabled": true, "message": "down", "dismissible": "no" }), + ] { + assert!( + validate_instance_banner(&bad).is_err(), + "{bad} should be rejected" + ); + } + // The strip is one or two lines tall; both of its texts are bounded. + for (field, over) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN + 1), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN + 1), + ] { + let mut banner = serde_json::json!({ "enabled": true, "message": "down" }); + banner[field] = serde_json::Value::String("x".repeat(over)); + assert!( + validate_instance_banner(&banner).is_err(), + "an over-long {field} should be rejected" + ); + } + // Enabled with no message renders as nothing and must stay writable: every path + // into this validator is a bulk settings save, so rejecting it would fail an + // admin's unrelated edits over a half-typed announcement. + assert!(validate_instance_banner(&serde_json::json!({ "enabled": true })).is_ok()); + let ok = serde_json::json!({ + "enabled": true, "message": "down", "severity": "warning", + "link": "https://status.example.com", "dismissible": false + }); + assert!(validate_instance_banner(&ok).is_ok()); + } + #[test] fn webhook_base_url_matches_the_ui_validator() { // Kept in lockstep with `isValidWebhookBaseUrl` in diff --git a/backend/windmill-common/src/guest_jwt.rs b/backend/windmill-common/src/guest_jwt.rs new file mode 100644 index 0000000000..9f3e47d708 --- /dev/null +++ b/backend/windmill-common/src/guest_jwt.rs @@ -0,0 +1,1017 @@ +//! The guest JWT contract: what a token minted by an embedding customer's own backend +//! must carry to open one guest-mode app, and how it is verified against the workspace's +//! configured key (or, off cloud, the instance issuer). Deliberately narrower than the external JWT scheme +//! (`jwt_ext_`), whose claims can assert admin, groups and folders: a guest key can +//! only ever mint guests, whatever the token says. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use jsonwebtoken::{ + jwk::{AlgorithmParameters, Jwk, PublicKeyUse}, + Algorithm, DecodingKey, Validation, +}; +use quick_cache::sync::Cache; +use serde::Deserialize; + +use crate::error::{Error, Result}; +use crate::DB; + +/// A token is honoured at most this long past its issue, however far its `exp` lies: +/// a guest's expiry is its only revocation, and a long-lived token minted by mistake +/// would otherwise stay valid until it leaked. +pub const MAX_LIFETIME_SECS: u64 = 24 * 60 * 60; + +/// Bearer prefix. Stateless: no `token` row. Verified against the workspace's key (or, off +/// cloud, the instance issuer when the workspace set none) and resolved in the auth cache, +/// whose entry is short-lived (not the token's full `exp`) so a rotated key revokes within +/// minutes. See the arm in `windmill-api-auth`. +pub const BEARER_PREFIX: &str = "jwt_guest_"; + +const RSA_ALGORITHMS: [Algorithm; 6] = [ + Algorithm::RS256, + Algorithm::RS384, + Algorithm::RS512, + Algorithm::PS256, + Algorithm::PS384, + Algorithm::PS512, +]; +const EC_ALGORITHMS: [Algorithm; 2] = [Algorithm::ES256, Algorithm::ES384]; + +/// Every claim honoured. Extra claims are ignored; a missing one refuses the token. +/// `app_path` is mandatory: a token opens that one app, exactly as a signed-in guest +/// session does. +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct GuestJwtClaims { + pub email: String, + pub workspace_id: String, + pub app_path: String, + pub exp: u64, + pub nbf: Option, + pub iat: Option, +} + +/// How a guest JWT is verified: the workspace's configured key (a PEM public key or a JWKS +/// URL), or, off cloud, the instance issuer (`JWT_EXT_JWKS_URL`) when the workspace set none — +/// see `key_source`. When neither is set the JWT is refused whatever it carries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GuestJwtKeySource { + Pem(String), + JwksUrl(String), +} + +/// The instance-wide external JWT issuer (`JWT_EXT_JWKS_URL`, also used by `jwt_ext_`). Read +/// fresh rather than cached so it is testable and picks up config regardless of init order. +fn instance_ext_jwks_url() -> Option { + std::env::var("JWT_EXT_JWKS_URL") + .ok() + .filter(|s| !s.trim().is_empty()) +} + +pub async fn key_source(db: &DB, w_id: &str) -> Result> { + let row = sqlx::query!( + "SELECT guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("reading guest JWT key of {w_id}: {e:#}")))?; + let per_workspace = row.and_then(|r| match (r.guest_jwt_public_key, r.guest_jwt_jwks_url) { + (Some(pem), _) => Some(GuestJwtKeySource::Pem(pem)), + (None, Some(url)) => Some(GuestJwtKeySource::JwksUrl(url)), + (None, None) => None, + }); + if per_workspace.is_some() { + return Ok(per_workspace); + } + // No workspace key: fall back to the instance issuer, so an operator running one issuer for + // both `jwt_ext_` and guests configures it once. Verifying it (and granting a *guest*) is + // done here in CE; granting a full login from it stays EE (`jwt_ext_`). Not on the shared + // cloud, where one instance issuer must not be trusted to mint guests in every tenant's + // workspace — there the per-workspace key is the only source. + if !*crate::worker::CLOUD_HOSTED { + if let Some(url) = instance_ext_jwks_url() { + return Ok(Some(GuestJwtKeySource::JwksUrl(url))); + } + } + Ok(None) +} + +/// Parse a PEM public key and the algorithms it may verify: RSA keys the RS/PS family, +/// EC keys the ES family. Anything symmetric has no PEM form, so HS* is unreachable +/// from here by construction; the JWKS path refuses it explicitly. +pub fn decoding_key_from_pem(pem: &str) -> Result<(DecodingKey, &'static [Algorithm])> { + use base64::{engine::general_purpose::STANDARD, Engine}; + use spki::der::Decode; + // The key is admin-set into an unbounded `TEXT` column and reparsed on every guest-JWT + // request; a well-formed key with an oversized modulus would pass the checks below. Refuse + // one larger than any real public key before decoding or storing it. Measure the untrimmed + // input: the endpoint stores what the admin sent, so whitespace padding counts too. + if pem.len() > MAX_GUEST_PEM_LEN { + return Err(Error::BadRequest(format!( + "guest key is longer than {MAX_GUEST_PEM_LEN} bytes" + ))); + } + let pem = pem.trim(); + // A verification key must be public. jsonwebtoken 8.3 keys the public/private distinction + // off the PEM label alone and never inspects the DER, so private material relabelled + // `PUBLIC KEY` would be stored and then served back through the settings response. Decode + // the body leniently, as jsonwebtoken does (tolerating any wrapping the strict RFC 7468 + // decoder would refuse), then require it to be a public-key structure: an SPKI (RSA or EC) + // or a PKCS#1 RSA public key. Private-key DER satisfies neither. + let der = STANDARD + .decode( + pem.lines() + .filter(|l| !l.trim_start().starts_with("-----")) + .flat_map(|l| l.split_whitespace()) + .collect::(), + ) + .map_err(|e| Error::BadRequest(format!("guest key is not valid PEM: {e}")))?; + let is_public = spki::SubjectPublicKeyInfoRef::from_der(&der).is_ok() + || pkcs1::RsaPublicKey::from_der(&der).is_ok(); + if !is_public { + return Err(Error::BadRequest( + "expected an RSA or EC public key in PEM form (-----BEGIN PUBLIC KEY-----)".to_string(), + )); + } + if let Ok(key) = DecodingKey::from_rsa_pem(pem.as_bytes()) { + return Ok((key, &RSA_ALGORITHMS)); + } + if let Ok(key) = DecodingKey::from_ec_pem(pem.as_bytes()) { + return Ok((key, &EC_ALGORITHMS)); + } + Err(Error::BadRequest( + "not an RSA or EC public key in PEM form (expected -----BEGIN PUBLIC KEY-----)".to_string(), + )) +} + +/// The algorithms a JWKS key may verify, or `None` if the key is unusable here: a +/// symmetric key (HS*, a shared secret the embedder would then have to hold), an +/// unsupported family, or a key not marked for signatures. A key that names its `alg` +/// pins that one; an RSA key that omits it accepts the whole RSA family, and an EC key +/// the algorithm its curve implies, mirroring how a PEM key is accepted. +pub fn jwk_algorithms(jwk: &Jwk) -> Option> { + if jwk.common.public_key_use.is_some() + && jwk.common.public_key_use != Some(PublicKeyUse::Signature) + { + return None; + } + // A key that lists its operations must allow verifying signatures; otherwise it is + // published for something else (encryption, key wrapping) and is not ours to use. + if jwk + .common + .key_operations + .as_ref() + .is_some_and(|ops| !ops.contains(&jsonwebtoken::jwk::KeyOperations::Verify)) + { + return None; + } + match (&jwk.algorithm, jwk.common.algorithm) { + (AlgorithmParameters::RSA(_), Some(alg)) if RSA_ALGORITHMS.contains(&alg) => { + Some(vec![alg]) + } + (AlgorithmParameters::RSA(_), None) => Some(RSA_ALGORITHMS.to_vec()), + (AlgorithmParameters::EllipticCurve(_), Some(alg)) if EC_ALGORITHMS.contains(&alg) => { + Some(vec![alg]) + } + (AlgorithmParameters::EllipticCurve(p), None) => match p.curve { + jsonwebtoken::jwk::EllipticCurve::P256 => Some(vec![Algorithm::ES256]), + jsonwebtoken::jwk::EllipticCurve::P384 => Some(vec![Algorithm::ES384]), + _ => None, + }, + _ => None, + } +} + +/// Verify `token` against `key`, honouring only the accepted `algorithms`, and check +/// every claim rule that needs no database: signature, `exp` (mandatory), `nbf` and +/// `iat` when present, the lifetime cap, that the token names `w_id`, that `email` is a +/// valid address bounded to 254 bytes, and that `app_path` carries no scope metacharacter. +pub fn verify( + token: &str, + key: &DecodingKey, + algorithms: &[Algorithm], + w_id: &str, +) -> Result { + let mut validation = Validation::new(algorithms[0]); + validation.algorithms = algorithms.to_vec(); + validation.validate_nbf = true; + let claims = jsonwebtoken::decode::(token, key, &validation) + .map_err(|e| Error::NotAuthorized(format!("guest JWT refused: {e}")))? + .claims; + let now = jsonwebtoken::get_current_timestamp(); + if claims.exp > now + MAX_LIFETIME_SECS { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: exp is more than {MAX_LIFETIME_SECS} seconds ahead" + ))); + } + if let Some(iat) = claims.iat { + if iat > now + validation.leeway { + return Err(Error::NotAuthorized( + "guest JWT refused: iat is in the future".to_string(), + )); + } + if claims.exp.saturating_sub(iat) > MAX_LIFETIME_SECS { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: lifetime exceeds {MAX_LIFETIME_SECS} seconds" + ))); + } + } + if claims.workspace_id != w_id { + return Err(Error::NotAuthorized( + "guest JWT refused: workspace_id does not match the workspace".to_string(), + )); + } + // The email becomes the guest's username; require the address shape the `usr` table + // accepts (`VALID_EMAIL`), so it always carries an `@` and `username_to_permissioned_as` + // can only ever read it as its own principal, never a `u/` or `g/`. + // Bound it to fit the `guest_activity.email` column: a longer one fails that insert + // while the guest is admitted uncounted. + if !crate::users::VALID_EMAIL.is_match(&claims.email) || claims.email.len() > 254 { + return Err(Error::NotAuthorized( + "guest JWT refused: email is not a valid, bounded email address".to_string(), + )); + } + // The app path is spliced into `apps:read:` and `apps:run:` scopes, whose + // grammar reserves `:`, `,`, `*` and a leading `/`; refuse those, the same guard + // `guest_session_scopes` applies at the mint. App paths may carry spaces and `@`. + if !crate::auth::is_scope_literal_path(&claims.app_path) { + return Err(Error::NotAuthorized( + "guest JWT refused: app_path is empty or cannot be scoped (`:`, `,`, `*` are \ + reserved and a leading `/` never matches a route)" + .to_string(), + )); + } + Ok(claims) +} + +struct JwksEntry { + keys: Arc>, + /// When this entry stops being served and the next request refetches. A good fetch + /// is served for `JWKS_TTL`, a failed one for `JWKS_NEGATIVE_TTL` (serving the last + /// good keys if there are any), so an unreachable issuer cannot be turned into one + /// outbound fetch per request by unauthenticated traffic. + expires_at: Instant, + /// When these keys were last fetched successfully. Stale keys are served only within + /// `JWKS_MAX_STALE` of this, and a stale re-serve preserves it, so a revoked `kid` or an + /// unreachable issuer stops minting new guest JWTs after a bounded window, not forever. + fetched_at: Instant, +} + +lazy_static::lazy_static! { + static ref JWKS_CACHE: Cache> = Cache::new(200); + /// Per-URL fetch lock: only one refresh per URL is in flight at a time, so a cold + /// or stale entry under a burst triggers one fetch, not one per request. This is a + /// plain map, not a capacity-bounded `Cache`: a `Cache` could evict a lock whose fetch + /// is still running, and the next request for that URL would then mint a fresh lock and + /// start a duplicate fetch, so cycling past 200 cold URLs could defeat single-flight and + /// storm the issuers. `JwksFetchLock` drops each entry once its last holder is gone, so + /// the map only ever holds the fetches in flight (bounded by concurrent distinct URLs). + static ref JWKS_FETCH_LOCKS: std::sync::Mutex>>> = + std::sync::Mutex::new(HashMap::new()); +} + +/// How long a good key set is served before a refresh; also the lag before a +/// rotated-in `kid` is picked up. The cadence of the instance-level external JWKS. +const JWKS_TTL: Duration = Duration::from_secs(15 * 60); +/// How long a failed fetch is remembered before retrying, so an unreachable issuer is +/// hit at most once per this interval however much guest-JWT traffic arrives. +const JWKS_NEGATIVE_TTL: Duration = Duration::from_secs(30); +/// The absolute age past which cached keys are no longer served, even while revalidating: +/// once an issuer has been unreachable (or has revoked a `kid`) for this long, its old keys +/// stop authenticating and the request fails closed rather than trusting them indefinitely. +const JWKS_MAX_STALE: Duration = Duration::from_secs(60 * 60); +/// A JWKS body larger than this is refused rather than buffered: the URL is admin-set +/// but the server it names may be attacker-controlled, and a real key set is a few KB. +const JWKS_MAX_BYTES: usize = 1 << 20; + +/// A cache entry retains only the usable signing keys, but nothing else bounds their combined +/// size: the response cap is 1 MiB and `from_jwk` decodes `n`/`e`/`x`/`y` without a length +/// limit, so one entry could retain ~1 MiB (a few hundred MB across the 200-entry LRU). Cap the +/// retained material instead; a real set is a few KB, so this is invisible to a legitimate one. +const JWKS_MAX_RETAINED_BYTES: usize = 64 * 1024; + +/// The retained-bytes cap counts key material; this caps the number of keys so the per-key +/// fixed cost (each `Jwk` and its map slot) is bounded too, not just their string content. +const JWKS_MAX_KEYS: usize = 50; + +/// Both JWKS caches key on the admin-supplied URL string. The column is unbounded `TEXT`, so +/// without this a workspace admin could grow the caches by the URL bytes alone. Enforced in +/// `fetch_jwks`, which `edit_guest_jwt_key` validates through, so an overlong URL is never +/// stored; the cache only ever sees a URL that was stored, hence a bounded one. +const MAX_JWKS_URL_LEN: usize = 2048; + +/// A guest verification key is admin-set into an unbounded `TEXT` column and reparsed on every +/// guest-JWT request. A real public key PEM is under a few KB (RSA-16384 SPKI is ~2.8 KB), so +/// this bounds the stored and reparsed bytes without refusing any real key. +const MAX_GUEST_PEM_LEN: usize = 8 * 1024; + +/// A guest JWT is refused past this before any signature work or caching: the auth cache keys +/// on the bearer, so an oversized token (unauthenticated at this point) would otherwise be +/// decoded and, if it verified, cached at its full size. A real JWT is well under this. +pub const MAX_GUEST_JWT_LEN: usize = 8 * 1024; + +/// Fetch a JWKS, keeping only the keys usable here. A workspace-admin URL is validated +/// against private ranges and the connect pinned to the validated addresses; redirects are +/// not followed for the same reason. The instance issuer (`JWT_EXT_JWKS_URL`) is exempt from +/// those restrictions — it is operator-configured and trusted (it also backs `jwt_ext_`), so a +/// self-hosted internal (http/private) issuer that works for `jwt_ext_` works for guests too. +/// The body is read with a cap so a hostile endpoint cannot exhaust memory. +pub async fn fetch_jwks(url: &str) -> Result> { + use futures::StreamExt; + if url.len() > MAX_JWKS_URL_LEN { + return Err(Error::BadRequest(format!( + "JWKS URL is longer than {MAX_JWKS_URL_LEN} bytes" + ))); + } + let resp = if instance_ext_jwks_url().as_deref() == Some(url) { + // Operator-trusted issuer: fetch it with the same permissive client `jwt_ext_` uses + // (follows redirects, honors ACCEPT_INVALID_CERTS), so an issuer that works for + // `jwt_ext_` through a redirect or an approved self-signed cert works for guests too. + crate::utils::HTTP_CLIENT_PERMISSIVE.get(url).send().await + } else { + // Workspace-admin URL: validate (https + private ranges), pin the connect to the + // validated addresses, and do not follow redirects — all against SSRF. + let client = crate::ssrf::validate_guest_jwks_url(url) + .await + .map_err(|e| Error::BadRequest(format!("JWKS URL is not allowed: {e}")))? + .apply_dns_pinning(crate::utils::configure_client(reqwest::ClientBuilder::new())) + .user_agent("windmill/beta") + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(10)) + .build() + .map_err(|e| Error::internal_err(format!("building JWKS client: {e}")))?; + client.get(url).send().await + } + .and_then(|r| r.error_for_status()) + .map_err(|e| Error::BadRequest(format!("could not fetch JWKS: {e}")))?; + let mut stream = resp.bytes_stream(); + let mut body: Vec = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| Error::BadRequest(format!("reading JWKS: {e}")))?; + if body.len() + chunk.len() > JWKS_MAX_BYTES { + return Err(Error::BadRequest(format!( + "JWKS is larger than {JWKS_MAX_BYTES} bytes" + ))); + } + body.extend_from_slice(&chunk); + } + parse_jwks_keys(&body) +} + +/// The usable signing keys in a JWKS body, by `kid`. Each key is parsed on its own and +/// one that does not model as a JWT key is skipped, not fatal: a set may legitimately +/// carry an encryption key (say `alg: "RSA-OAEP"`, which is not in jsonwebtoken's signing +/// `Algorithm` enum and would fail whole-set deserialization) beside its signing keys. +/// +/// A key is kept only if its material actually decodes (`DecodingKey::from_jwk`): jsonwebtoken +/// carries `n`/`e`/`x`/`y` as strings and defers decoding to auth time, so without this a JWKS +/// whose only key is malformed would be accepted at save time and fail every token later. +fn parse_jwks_keys(body: &[u8]) -> Result> { + let set: serde_json::Value = serde_json::from_slice(body) + .map_err(|e| Error::BadRequest(format!("JWKS is not JSON: {e}")))?; + let entries = set + .get("keys") + .and_then(|k| k.as_array()) + .ok_or_else(|| Error::BadRequest("JWKS has no `keys` array".to_string()))?; + let keys: HashMap = entries + .iter() + .filter_map(|entry| serde_json::from_value::(entry.clone()).ok()) + .filter(|jwk| jwk_algorithms(jwk).is_some()) + .filter(|jwk| DecodingKey::from_jwk(jwk).is_ok()) + .filter_map(|jwk| jwk.common.key_id.clone().map(|kid| (kid, jwk))) + .collect(); + if keys.is_empty() { + return Err(Error::BadRequest( + "JWKS holds no RSA or EC signing key with a kid".to_string(), + )); + } + // Bound the usable keys two ways, both measured after filtering so a large mixed-use set + // (many encryption keys, few signing) is not refused for its size: their count (the per-key + // fixed cost) and their combined material bytes. + if keys.len() > JWKS_MAX_KEYS { + return Err(Error::BadRequest(format!( + "JWKS holds more than {JWKS_MAX_KEYS} usable signing keys" + ))); + } + let retained: usize = keys + .values() + .filter_map(|jwk| serde_json::to_vec(jwk).ok().map(|v| v.len())) + .sum(); + if retained > JWKS_MAX_RETAINED_BYTES { + return Err(Error::BadRequest(format!( + "JWKS signing keys retain more than {JWKS_MAX_RETAINED_BYTES} bytes" + ))); + } + Ok(keys) +} + +/// A cached entry that holds no keys is a remembered failure; serving it would report +/// an unreachable issuer as an unknown `kid`. Map it to an issuer-unreachable error. +fn servable(entry: Arc) -> Result> { + if entry.keys.is_empty() { + Err(Error::NotAuthorized( + "guest JWT refused: the JWKS issuer is unreachable".to_string(), + )) + } else { + Ok(entry) + } +} + +/// A held single-flight lock for one JWKS URL. Dropping it removes the URL from +/// `JWKS_FETCH_LOCKS` once no other holder remains, so the registry never keeps a lock past +/// its fetch and stays bounded by the number of fetches in flight, not by URLs ever seen. +struct JwksFetchLock { + url: String, + lock: Arc>, +} + +impl JwksFetchLock { + /// The lock for `url`, created on first use. Callers sharing a URL get the same `Arc`, + /// so one holds the inner mutex and fetches while the rest wait on it. + fn acquire(url: &str) -> Self { + let mut map = JWKS_FETCH_LOCKS.lock().unwrap(); + let lock = map + .entry(url.to_string()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone(); + JwksFetchLock { url: url.to_string(), lock } + } +} + +impl Drop for JwksFetchLock { + fn drop(&mut self) { + let mut map = JWKS_FETCH_LOCKS.lock().unwrap(); + // Clones are only taken while holding this same map lock, so the count is stable + // here: two Arcs (the map's and this one's) means we are the last holder and the + // entry can go; more means another request still needs it and will remove it in turn. + if map + .get(&self.url) + .is_some_and(|lock| Arc::strong_count(lock) <= 2) + { + map.remove(&self.url); + } + } +} + +/// Refresh a URL's JWKS off the request path, under the single-flight lock. A held +/// lock means a refresh is already running, so this is a no-op. A failed refresh +/// leaves the served stale keys in place rather than dropping them. +fn spawn_jwks_refresh(url: String) { + tokio::spawn(async move { + let fetch_lock = JwksFetchLock::acquire(&url); + let Ok(_guard) = fetch_lock.lock.try_lock() else { + return; + }; + match fetch_jwks(&url).await { + Ok(keys) => { + let now = Instant::now(); + JWKS_CACHE.insert( + url, + Arc::new(JwksEntry { + keys: Arc::new(keys), + expires_at: now + JWKS_TTL, + fetched_at: now, + }), + ); + } + Err(e) => tracing::warn!("guest JWKS background refresh failed for {url}: {e:#}"), + } + }); +} + +/// The workspace's JWKS. A fresh entry is served directly; a stale-but-good one is +/// served while a refresh runs off the request path (`spawn_jwks_refresh`), so a slow +/// issuer never stalls a request. Only a cold or negative entry blocks, under a per-URL +/// lock so a burst triggers one fetch; a failed fetch there caches a short-lived empty +/// entry that reads as "issuer unreachable", so an unreachable issuer is hit at most +/// once per `JWKS_NEGATIVE_TTL`. Fetches follow a schedule, never a per-request, +/// attacker-chosen `kid`. +async fn cached_jwks(url: &str) -> Result> { + if let Some(entry) = JWKS_CACHE.get(url) { + // Keys past JWKS_MAX_STALE are never served, even while revalidating: a stale re-serve + // bumps expires_at but keeps fetched_at, so a persistently failing refresh would + // otherwise serve revoked keys forever. Too-old keys fall through to the blocking + // refresh, which fails closed if the issuer is still down. + if entry.fetched_at.elapsed() < JWKS_MAX_STALE { + if entry.expires_at > Instant::now() { + return servable(entry); + } + // Stale but still holds good keys: serve them now and refresh off the request + // path, so a slow or hanging issuer adds no latency. Bump the entry first so the + // refresh window does not spawn a task per request. A negative (empty) entry + // falls through to the blocking refresh below. + if !entry.keys.is_empty() { + let served = Arc::new(JwksEntry { + keys: entry.keys.clone(), + expires_at: Instant::now() + JWKS_NEGATIVE_TTL, + fetched_at: entry.fetched_at, + }); + JWKS_CACHE.insert(url.to_string(), served.clone()); + spawn_jwks_refresh(url.to_string()); + return Ok(served); + } + } + } + // Cold or negative entry, nothing good to serve: block on a single-flight refresh. + // `JwksFetchLock::acquire` hands cold requests the same lock, so they share one fetch. + let fetch_lock = JwksFetchLock::acquire(url); + let _guard = fetch_lock.lock.lock().await; + // Another task may have refreshed while we waited for the lock (honour the age limit too, + // so a concurrent stale re-serve of too-old keys is not mistaken for a fresh entry). + if let Some(entry) = JWKS_CACHE.get(url) { + if entry.fetched_at.elapsed() < JWKS_MAX_STALE && entry.expires_at > Instant::now() { + return servable(entry); + } + } + match fetch_jwks(url).await { + Ok(keys) => { + let now = Instant::now(); + let entry = Arc::new(JwksEntry { + keys: Arc::new(keys), + expires_at: now + JWKS_TTL, + fetched_at: now, + }); + JWKS_CACHE.insert(url.to_string(), entry.clone()); + Ok(entry) + } + Err(e) => { + // Reached with no servable keys: either nothing cached, or keys too old to trust. + // Cache a short negative entry so the next requests do not each refetch, and + // surface the error, so an issuer that revoked a key or went down fails closed. + JWKS_CACHE.insert( + url.to_string(), + Arc::new(JwksEntry { + keys: Arc::new(HashMap::new()), + expires_at: Instant::now() + JWKS_NEGATIVE_TTL, + fetched_at: Instant::now(), + }), + ); + Err(e) + } + } +} + +/// The key a token's header selects from the workspace's JWKS, by `kid`, and the +/// algorithms it may verify. An unknown `kid` is refused against the cached set rather +/// than triggering a fetch, so varying `kid` cannot drive outbound requests; a +/// genuinely rotated-in key is picked up within `JWKS_TTL`. +pub async fn jwks_key_for(url: &str, token: &str) -> Result<(DecodingKey, Vec)> { + let header = jsonwebtoken::decode_header(token) + .map_err(|e| Error::NotAuthorized(format!("guest JWT refused: {e}")))?; + let kid = header.kid.ok_or_else(|| { + Error::NotAuthorized("guest JWT refused: no kid in the header".to_string()) + })?; + let entry = cached_jwks(url).await?; + let jwk = entry.keys.get(&kid).ok_or_else(|| { + Error::NotAuthorized(format!("guest JWT refused: kid {kid} is not in the JWKS")) + })?; + let algs = jwk_algorithms(jwk).ok_or_else(|| { + Error::NotAuthorized(format!("guest JWT refused: kid {kid} is not a signing key")) + })?; + let key = DecodingKey::from_jwk(jwk) + .map_err(|e| Error::internal_err(format!("unusable JWK {kid}: {e}")))?; + Ok((key, algs)) +} + +/// Verify `token` for `w_id` against whatever key the workspace configured. A PEM key +/// ignores `kid`; a JWKS selects by it. +pub async fn verify_for_workspace(db: &DB, w_id: &str, token: &str) -> Result { + // The admit check downstream refuses these anyway; refusing here keeps a deployment + // with no guests from parsing attacker-supplied JWTs at all, and names the reason in + // the log the caller writes. + if !crate::workspaces::instance_supports_guests() { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: {}", + crate::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))); + } + if token.len() > MAX_GUEST_JWT_LEN { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: token is longer than {MAX_GUEST_JWT_LEN} bytes" + ))); + } + let Some(source) = key_source(db, w_id).await? else { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: workspace {w_id} has no guest JWT key" + ))); + }; + match source { + GuestJwtKeySource::Pem(pem) => { + let (key, algorithms) = decoding_key_from_pem(&pem)?; + verify(token, &key, algorithms, w_id) + } + GuestJwtKeySource::JwksUrl(url) => { + let (key, algorithms) = jwks_key_for(&url, token).await?; + verify(token, &key, &algorithms, w_id) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Serializes the tests that mutate the process-wide `ALLOW_PRIVATE_GUEST_JWKS_URLS`, so a + /// concurrent run cannot clear it out from under another (mirrors `ssrf.rs`'s test lock). + static TEST_ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + fn jwk(v: serde_json::Value) -> Jwk { + serde_json::from_value(v).unwrap() + } + + #[test] + fn rsa_key_with_alg_pins_it() { + let k = jwk(serde_json::json!({"kty":"RSA","alg":"RS384","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), Some(vec![Algorithm::RS384])); + } + + #[test] + fn rsa_key_without_alg_takes_the_whole_family() { + // The bug this pins: an alg-less RSA key must not be forced to RS256, which + // would reject valid RS384/512 or PS* tokens. + let k = jwk(serde_json::json!({"kty":"RSA","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), Some(RSA_ALGORITHMS.to_vec())); + } + + #[test] + fn ec_key_takes_its_curve_algorithm() { + let k = jwk(serde_json::json!({"kty":"EC","crv":"P-256","x":"aa","y":"bb"})); + assert_eq!(jwk_algorithms(&k), Some(vec![Algorithm::ES256])); + } + + #[test] + fn symmetric_key_is_refused() { + let k = jwk(serde_json::json!({"kty":"oct","k":"c2VjcmV0"})); + assert_eq!(jwk_algorithms(&k), None); + } + + #[test] + fn a_key_marked_for_encryption_is_refused() { + let k = jwk(serde_json::json!({"kty":"RSA","use":"enc","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), None); + } + + #[test] + fn a_mixed_use_jwks_keeps_only_the_signing_keys() { + // An encryption key (RSA-OAEP is not in jsonwebtoken's signing Algorithm enum) + // beside a signing key must not fail the whole set. The signing key carries real + // coordinates so it survives the material check parse_jwks_keys now applies. + let body = serde_json::json!({ + "keys": [ + {"kty":"RSA","alg":"RSA-OAEP","kid":"enc","use":"enc","n":"aa","e":"AQAB"}, + {"kty":"EC","crv":"P-256","kid":"sig","x":PUB1_X,"y":PUB1_Y} + ] + }) + .to_string(); + let keys = parse_jwks_keys(body.as_bytes()).expect("the signing key survives"); + assert!(keys.contains_key("sig")); + assert!(!keys.contains_key("enc")); + } + + #[test] + fn a_jwks_with_only_malformed_key_material_is_refused() { + // Metadata (kty/alg/use) is fine but `n` is not valid base64url, so the key is + // unusable. Since it is the only key, configuring this JWKS must fail at save time + // rather than persist a URL whose tokens all fail later. + let body = serde_json::json!({ + "keys": [{"kty":"RSA","kid":"k1","n":"not base64url!!","e":"AQAB"}] + }) + .to_string(); + assert!(parse_jwks_keys(body.as_bytes()).is_err()); + } + + #[test] + fn too_many_usable_keys_is_refused() { + // All keys are usable (real coordinates), so this trips the count cap, not the + // empty-set path. Small material, so it is the count that refuses them, not the bytes. + let keys: Vec<_> = (0..=JWKS_MAX_KEYS) + .map(|i| { + serde_json::json!({"kty":"EC","crv":"P-256","kid":format!("k{i}"),"x":PUB1_X,"y":PUB1_Y}) + }) + .collect(); + let body = serde_json::json!({ "keys": keys }).to_string(); + let err = parse_jwks_keys(body.as_bytes()).unwrap_err().to_string(); + assert!(err.contains("usable signing keys"), "{err}"); + } + + #[test] + fn keys_retaining_too_many_bytes_is_refused() { + // Under the key count cap, but the material of these usable RSA keys exceeds the byte + // cap. `from_jwk` decodes any-length `n`, so this is reachable without the count cap. + let big_n = "A".repeat(2000); + let keys: Vec<_> = (0..40) + .map(|i| serde_json::json!({"kty":"RSA","kid":format!("k{i}"),"n":big_n,"e":"AQAB"})) + .collect(); + let body = serde_json::json!({ "keys": keys }).to_string(); + let err = parse_jwks_keys(body.as_bytes()).unwrap_err().to_string(); + assert!(err.contains("retain more than"), "{err}"); + } + + #[tokio::test] + async fn an_overlong_jwks_url_is_refused() { + // The cache keys on the URL string, so an unbounded URL is refused before it is cached. + // Assert the length error specifically: a bogus URL would fail the fetch regardless. + let url = format!( + "https://issuer.example.com/{}", + "a".repeat(MAX_JWKS_URL_LEN) + ); + let err = fetch_jwks(&url).await.unwrap_err().to_string(); + assert!(err.contains("longer than"), "{err}"); + } + + #[test] + fn an_oversized_pem_is_refused() { + // A well-formed key body padded past the cap: refused for its length before decoding, + // so an oversized-but-valid key cannot be stored and reparsed on every request. + let big = format!( + "-----BEGIN PUBLIC KEY-----\n{}\n-----END PUBLIC KEY-----\n", + "A".repeat(MAX_GUEST_PEM_LEN) + ); + let err = decoding_key_from_pem(&big).err().unwrap().to_string(); + assert!(err.contains("longer than"), "{err}"); + // Whitespace padding must count: the endpoint stores the untrimmed value, so the cap is + // measured before trimming rather than on the small trimmed key it would otherwise see. + let padded = format!("{}{RSA_PUBLIC}", " ".repeat(MAX_GUEST_PEM_LEN)); + let err = decoding_key_from_pem(&padded).err().unwrap().to_string(); + assert!(err.contains("longer than"), "{err}"); + } + + #[test] + fn key_ops_without_verify_is_refused() { + let enc = jwk(serde_json::json!({"kty":"RSA","key_ops":["encrypt"],"n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&enc), None); + let ver = jwk(serde_json::json!({"kty":"RSA","key_ops":["verify"],"n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&ver), Some(RSA_ALGORITHMS.to_vec())); + } + + // PUB1's coordinates and its matching PKCS8 private key, for the JWKS-derived + // verification test. + const PUB1_X: &str = "zAfqyCh34iYOCW0vg4ejq_zzJlzLSZScjnVyPjLGTao"; + const PUB1_Y: &str = "RMKOIHOv8tWLnXf7-eMCodDnX038wCjD1sf9jVsf7oI"; + const PRIV1: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgu27S2DbSwUh8BmQb\n/i4/VhNdoXV7PJekhnoceMULYLihRANCAATMB+rIKHfiJg4JbS+Dh6Or/PMmXMtJ\nlJyOdXI+MsZNqkTCjiBzr/LVi513+/njAqHQ519N/MAow9bH/Y1bH+6C\n-----END PRIVATE KEY-----\n"; + + #[test] + fn a_jwks_key_verifies_a_real_token() { + // The one path the PEM tests do not cover: a key rebuilt from a JWK verifies a + // token signed by its private half, and the algorithms come from the JWK. + let jwk = jwk(serde_json::json!({ + "kty": "EC", "crv": "P-256", "kid": "k1", "x": PUB1_X, "y": PUB1_Y + })); + let algs = jwk_algorithms(&jwk).expect("EC signing key"); + assert_eq!(algs, vec![Algorithm::ES256]); + let key = jsonwebtoken::DecodingKey::from_jwk(&jwk).expect("usable JWK"); + let payload = serde_json::json!({ + "email": "g@example.com", + "workspace_id": "ws", + "app_path": "u/a/app", + "exp": jsonwebtoken::get_current_timestamp() + 600, + }); + let token = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::ES256), + &payload, + &jsonwebtoken::EncodingKey::from_ec_pem(PRIV1.as_bytes()).unwrap(), + ) + .unwrap(); + let out = verify(&token, &key, &algs, "ws").expect("verifies"); + assert_eq!(out.email, "g@example.com"); + // The workspace pin is part of verify. + assert!(verify(&token, &key, &algs, "other-ws").is_err()); + } + + #[test] + fn a_non_key_pem_is_rejected() { + assert!(decoding_key_from_pem( + "-----BEGIN CERTIFICATE-----\nnope\n-----END CERTIFICATE-----" + ) + .is_err()); + } + + // A real RSA public key (SPKI). Its private counterpart is RSA_PKCS1_PRIVATE below. + const RSA_PUBLIC: &str = "-----BEGIN PUBLIC KEY-----\n\ +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx3J0fQcHp2ZlMI4rCVsY\n\ +tirATZPWyPD7exoYWPInhV5xjbY2Fe8IVFaZszQcQbCXZjBtFp2fj0tBTow8BeOy\n\ +X9LJPyKeho/j68FycuDVg7JCzG0TWtsnh/V23WkrlKIfmMqS3+YUyFavROTcAN1T\n\ +5BcFHLAi/4Q2qy0JjXBdZ8avelzZrQ/T67/Kcsoct/pvnEDT2YRsSbA7VMWaxWh8\n\ +MYJ7GNV/10YT2c5CBJGSLbyRSVWk2IwfnM9Cl9n/5NE6TkSetYQ2xlqKTONp5W43\n\ +UzW1NAeqKCxPQfN/ADjwW18nk2o7xj1kMF4rBlhsTm9ClE71nwi5NxsvMOdVxytZ\n\ +ZQIDAQAB\n\ +-----END PUBLIC KEY-----\n"; + + // A complete, valid PKCS#1 RSA *private* key (the counterpart of RSA_PUBLIC) with its + // armor relabelled `RSA PUBLIC KEY`. jsonwebtoken's from_rsa_pem accepts it under that + // label; the structural check refuses it (a 9-field RSAPrivateKey is neither an SPKI nor + // a 2-field RsaPublicKey). Complete on purpose: malformed DER would fail for the wrong + // reason and let a real bypass through unnoticed. + const RSA_PKCS1_PRIVATE_AS_PUBLIC: &str = "-----BEGIN RSA PUBLIC KEY-----\n\ +MIIEogIBAAKCAQEAx3J0fQcHp2ZlMI4rCVsYtirATZPWyPD7exoYWPInhV5xjbY2\n\ +Fe8IVFaZszQcQbCXZjBtFp2fj0tBTow8BeOyX9LJPyKeho/j68FycuDVg7JCzG0T\n\ +Wtsnh/V23WkrlKIfmMqS3+YUyFavROTcAN1T5BcFHLAi/4Q2qy0JjXBdZ8avelzZ\n\ +rQ/T67/Kcsoct/pvnEDT2YRsSbA7VMWaxWh8MYJ7GNV/10YT2c5CBJGSLbyRSVWk\n\ +2IwfnM9Cl9n/5NE6TkSetYQ2xlqKTONp5W43UzW1NAeqKCxPQfN/ADjwW18nk2o7\n\ +xj1kMF4rBlhsTm9ClE71nwi5NxsvMOdVxytZZQIDAQABAoIBAD+IbaQQM7d3Dj/X\n\ +4cyyqJ4K40QzFmXfIfTWXLAkv0MkUR7XzsXQ5YHcLkzgCipAwxGp1m4wWs4OJmkL\n\ +kek8XatZnYLPl9j8iBmm/zqp9Unk5JNzIYm9KwwLvMgOAvRvaopE6WGKTM9+kYls\n\ +L8rUti7/yECZuSRU7Qc9KwBTrWVrXK+RBtBqZYQXb92BFxq0N3Qp+utLNdFcO5sW\n\ +7d8gKp3ipQt5z9ZAB2pYMw7ZTzonF4C7HdyrbYXztvYrxuw1imMkQ9iFFhdn3/76\n\ +qFR7XwaFrld8DECGaH/652kV6zaSQijbBTeXF4zsgwXY4BHMVmZKaXH5unw8Gbmo\n\ +WCoLbLcCgYEA4vTo5kCiKXnlBp3W1Zpg4cml6Wzo0UDldF4kkuXQxhdQA38c0ise\n\ +Cocf6qyAqz1L2TxQ/9WCL2oIP1AY9XqnQ0cJtYIosGWORz4tPe67M8inB/GBotFI\n\ +pmQNVSIjqbgKVi0x+UzmFjitINFPf461lDdJTwhsv9TQRbHrXErvON8CgYEA4PhV\n\ +GYMJu46tqFVtD/koWAQRLmeaZXhxP5lMSmQjdYCa3ys5lccvTlzoF9K8immMQkIx\n\ +gyOazmEtFnK4IXmEY1wg2NIHuJM7/maoM2rozbjXBxsYM7Xw2QX7BHXqG6Ia/Bij\n\ +ZaRJdumCVRJv7OshQTGuqDIzd3l5WEqg11XYgjsCgYAP3v6Wc3ijm+GXN9x5LYWO\n\ +5JIUo8gYMgiZvaejGi0iXSj8RZxXWiqMo+xodc29q9itBVnIuj6TYD/ZZZmJOR2P\n\ +R9128vYzd7aeZsu1JAe1VFfR52KgZzBEaoTAKlYCHVujsR9ohqckcKwyulBr5Cfw\n\ +iHk47KbmN1SlOw7xclAOUwKBgAuxHEsdIk5bFe9fsTFZU51vaK0uuTl4zvntL6fW\n\ +GHms21+p0W5VUcIS1gUW8LGI1r9CzWvxV8RODJfUEnm65QR870AVek0/aajJEQjL\n\ +D5pRdutpnxJg7El7JBaRQj95Z0mexi8sIJ1LeXiOYr6/YZUPzfHz2fTlnUbXahCG\n\ +55+tAoGAI811NTb7kuuIPYuj4raDW88QVNX2xB3+p9lXGolB4jPgsUEjgSvLgH9S\n\ +Q/LwEBiCYVyii8MvWsIZpHvSGyOoty2p19/CAvrAOfpEVlnXQeiX+mh09p1mQbfM\n\ +y9rTR828ADcaZ63Ej1oL4GcqmGhODxCLy1YKKcy0FHzChqPMV6g=\n\ +-----END RSA PUBLIC KEY-----\n"; + + #[test] + fn a_private_pem_is_refused() { + // A verification key must be public and must never be stored otherwise: it is served + // back through the settings response. jsonwebtoken keys the public/private split off + // the PEM label, so private DER relabelled with a public armor slips a label check; + // only parsing the DER as a public-key structure (SPKI or PKCS#1 RSA public) refuses + // it. A real public key still parses, so the guard is not vacuous. + assert!(decoding_key_from_pem(RSA_PUBLIC).is_ok()); + // The same key with its body on one line (not 64-column wrapped): jsonwebtoken accepts + // any wrapping, so the guard must too rather than lean on the strict RFC 7468 decoder. + let body: String = RSA_PUBLIC + .lines() + .filter(|l| !l.starts_with("-----")) + .collect(); + let one_line = format!("-----BEGIN PUBLIC KEY-----\n{body}\n-----END PUBLIC KEY-----\n"); + assert!(decoding_key_from_pem(&one_line).is_ok()); + + assert!(decoding_key_from_pem(PRIV1).is_err()); + // The same PKCS#8 EC private key, relabelled `PUBLIC KEY`. + assert!(decoding_key_from_pem(&PRIV1.replace("PRIVATE KEY", "PUBLIC KEY")).is_err()); + assert!(decoding_key_from_pem(RSA_PKCS1_PRIVATE_AS_PUBLIC).is_err()); + } + + #[tokio::test] + async fn a_concurrent_cold_burst_makes_one_jwks_fetch() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::io::AsyncWriteExt; + let _env = TEST_ENV_LOCK.lock().await; + // The stub listens on loopback, which SSRF validation refuses without this. + unsafe { std::env::set_var("ALLOW_PRIVATE_GUEST_JWKS_URLS", "true") }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let body = format!( + r#"{{"keys":[{{"kty":"EC","crv":"P-256","kid":"k1","x":"{PUB1_X}","y":"{PUB1_Y}"}}]}}"# + ); + let hits = std::sync::Arc::new(AtomicUsize::new(0)); + let hits_srv = hits.clone(); + tokio::spawn(async move { + loop { + let (mut sock, _) = listener.accept().await.unwrap(); + hits_srv.fetch_add(1, Ordering::SeqCst); + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + // Delay so the other callers pile onto the single-flight lock before the + // leader's fetch returns. + tokio::time::sleep(Duration::from_millis(150)).await; + let _ = sock.write_all(resp.as_bytes()).await; + let _ = sock.shutdown().await; + } + }); + let url = format!("http://127.0.0.1:{}/jwks.json", addr.port()); + let mut handles = Vec::new(); + for _ in 0..10 { + let u = url.clone(); + handles.push(tokio::spawn(async move { + cached_jwks(&u).await.map(|e| e.keys.len()) + })); + } + for h in handles { + assert_eq!( + h.await.unwrap().unwrap(), + 1, + "each caller resolves the one key" + ); + } + assert_eq!( + hits.load(Ordering::SeqCst), + 1, + "single-flight: a concurrent cold burst makes one fetch" + ); + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + } + + #[tokio::test] + async fn jwks_fetch_locks_are_shared_and_self_cleaning() { + // The registry is a plain map, not a capacity-bounded cache: a cache could evict a + // lock mid-fetch, letting a later request for that URL start a duplicate fetch. Pin + // both halves of what keeps single-flight intact under many distinct URLs: the same + // URL hands back one shared lock, and the entry is removed once its last holder drops + // (so nothing evicts an in-flight lock and the map stays bounded by fetches in flight). + let url = "https://example.test/jwks-lock-probe.json"; + { + let a = JwksFetchLock::acquire(url); + let b = JwksFetchLock::acquire(url); + assert!(Arc::ptr_eq(&a.lock, &b.lock), "one lock per URL"); + assert!(JWKS_FETCH_LOCKS.lock().unwrap().contains_key(url)); + } + assert!( + !JWKS_FETCH_LOCKS.lock().unwrap().contains_key(url), + "the lock is dropped once idle" + ); + } + + #[tokio::test] + async fn stale_jwks_keys_stop_being_served_past_the_grace_window() { + let _env = TEST_ENV_LOCK.lock().await; + unsafe { std::env::set_var("ALLOW_PRIVATE_GUEST_JWKS_URLS", "true") }; + // A dead loopback port, so every refresh fails (connection refused). + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let jwk = jwk(serde_json::json!( + {"kty":"EC","crv":"P-256","kid":"k1","x":PUB1_X,"y":PUB1_Y} + )); + let keys: HashMap = [("k1".to_string(), jwk)].into_iter().collect(); + let stale = Instant::now().checked_sub(Duration::from_secs(1)).unwrap(); + + // Within the grace window, stale keys are still served while a (failing) refresh runs. + let within = format!("http://127.0.0.1:{port}/within"); + JWKS_CACHE.insert( + within.clone(), + Arc::new(JwksEntry { + keys: Arc::new(keys.clone()), + expires_at: stale, + fetched_at: Instant::now(), + }), + ); + assert!( + cached_jwks(&within).await.is_ok(), + "stale keys within the grace window are still served" + ); + + // Past the grace window, the keys are not served: the failing refresh fails closed. + let beyond = format!("http://127.0.0.1:{port}/beyond"); + JWKS_CACHE.insert( + beyond.clone(), + Arc::new(JwksEntry { + keys: Arc::new(keys), + expires_at: stale, + fetched_at: Instant::now() + .checked_sub(JWKS_MAX_STALE + Duration::from_secs(1)) + .unwrap(), + }), + ); + assert!( + cached_jwks(&beyond).await.is_err(), + "keys past the grace window fail closed once the refresh fails" + ); + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + } + + #[tokio::test] + async fn a_plaintext_http_jwks_url_is_refused_by_default() { + let _env = TEST_ENV_LOCK.lock().await; + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + // The JWKS supplies the keys that authenticate guest JWTs; without the operator opt-in, + // a plaintext URL (which an on-path attacker could replace) is refused for its scheme. + assert!(matches!( + crate::ssrf::validate_guest_jwks_url("http://issuer.example.com/jwks.json").await, + Err(crate::ssrf::SsrfValidationError::HttpsRequired) + )); + } + + #[tokio::test] + async fn the_instance_issuer_bypasses_the_https_and_private_restriction() { + let _env = TEST_ENV_LOCK.lock().await; + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + // The instance issuer is operator-trusted (it also backs jwt_ext_), so an http/private + // URL is not refused for its scheme: it reaches the connect and fails there (dead port), + // not at validation. A different URL is not the instance issuer and is still refused. + let instance = "http://127.0.0.1:1/jwks.json"; + unsafe { std::env::set_var("JWT_EXT_JWKS_URL", instance) }; + let trusted = fetch_jwks(instance).await.err().unwrap().to_string(); + let other = fetch_jwks("http://127.0.0.1:1/other.json") + .await + .err() + .unwrap() + .to_string(); + unsafe { std::env::remove_var("JWT_EXT_JWKS_URL") }; + assert!( + !trusted.contains("not allowed") && !trusted.contains("must use https"), + "instance issuer skips validation: {trusted}" + ); + assert!( + other.contains("not allowed") || other.contains("https"), + "a non-instance http url is still refused: {other}" + ); + } +} diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 7ea6dbe5ae..1dd3396809 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -916,6 +916,14 @@ pub struct WorkerGroupConfig { pub autoscaling: Option, #[serde(skip_serializing_if = "Option::is_none")] pub native_mode: Option, + /// Object store this group's dependency cache uses instead of the instance one. Same shape + /// as the instance `object_store_cache_config`. + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "instance_config_schema", + schemars(schema_with = "opaque_json_schema") + )] + pub object_store_cache_config: Option, /// Catch-all for fields not yet covered by typed fields. #[serde(flatten)] @@ -1280,8 +1288,9 @@ pub fn diff_worker_configs( ConfigsDiff { upserts, deletes } } -/// Declaratively replace the global settings, rejecting a `github_app_webhook_base_url` -/// the API would reject. +/// Declaratively replace the global settings, rejecting a +/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the +/// API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1330,6 +1339,23 @@ pub async fn sync_global_settings_declarative( } } + let banner_key = crate::global_settings::INSTANCE_BANNER_SETTING; + match desired.get(banner_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(banner) => crate::global_settings::validate_instance_banner(banner) + // The validator's messages name the offending field and its expected type, + // never the submitted value, so they are safe to surface here. + .map_err(|e| anyhow::anyhow!("{banner_key}: {e}"))?, + } + + // An origin list that cannot be parsed is dropped at boot, leaving the + // empty default — which is no restriction at all. Rejecting it here is what + // keeps a typo in a ConfigMap from silently widening CORS instance-wide. + let origins_key = crate::global_settings::HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING; + crate::global_settings::parse_allowed_origins_setting(desired.get(origins_key)) + .map_err(|e| anyhow::anyhow!("{origins_key}: {e}"))?; + let diff = diff_global_settings(current, desired, ApplyMode::Replace); apply_settings_diff(db, &diff).await?; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index b4128f80d2..dce40048f5 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -67,6 +67,7 @@ pub mod flow_status; pub mod flows; pub mod folders; pub mod global_settings; +pub mod guest_jwt; pub mod indexer; pub mod instance_config; pub mod job_metrics; @@ -110,6 +111,7 @@ pub use pipeline_advanced_ee as pipeline_advanced; pub use pipeline_advanced_oss as pipeline_advanced; pub mod query_builders; pub mod queue; +pub mod queue_metrics; pub mod result_stream; pub mod runnable_settings; pub mod schedule; @@ -280,14 +282,16 @@ pub fn check_on_behalf_of_preservation( None } -/// Resolves the identity to store when creating/updating a flow or script. +/// Resolves the identity to store when creating/updating a flow, script or app. /// -/// The permissioned_as is the only stored identity — it decides what the job may access, -/// and the address is derived from it at read time — so the two can never name different -/// accounts. Callers may supply either: a bare email (every client written before the -/// principal existed) is resolved to the principal it names, and an email that names -/// nobody is rejected rather than recorded, since it could only produce a runnable that -/// cannot authenticate. +/// The permissioned_as is the identity: it decides what the job may access, and the address is +/// a function of it, so the two can never name different accounts. For a script or flow the +/// address is derived at read time; an app still stores it, as a compatibility copy written +/// through from the principal on every save and returned verbatim by the app reads (see +/// `docs/app-policy-email-removal.md`). Callers may supply either: a bare email (every client +/// written before the principal existed) is resolved to the principal it names, and an email +/// that names nobody is rejected rather than recorded, since it could only produce a runnable +/// that cannot authenticate. /// /// Returns `None` when the runnable has no on-behalf-of identity, and the caller's own /// identity when they are not allowed to preserve someone else's. @@ -295,6 +299,18 @@ pub fn check_on_behalf_of_preservation( /// Resolves through the non-RLS pool and authorizes nothing itself — `authed` decides only /// whether preservation is allowed, and its role flags are not re-checked against `w_id`. /// Callers must already be authorized for the workspace they pass. +/// +/// Known, accepted race. The lookup runs on the pool, outside the caller's write transaction, so +/// an account renamed or removed between the two has its sweep run before the write is visible, +/// and the write stores the old principal. The runnable then fails to authenticate until it is +/// deployed with a current identity, with two exceptions: an app naming an external superadmin +/// keeps running as that account through its stored address, and if the freed username is later +/// given to another account, the stale principal binds to that account and runs as it. Every +/// caller shares this (scripts, flows and apps, address-only inputs included), and it needs a +/// rename or removal of the exact account inside the lookup-to-commit gap. Closing it means +/// serializing every identity write against every identity mutation, across all runnable kinds +/// (a `usr` row lock in each write, with each sweep ordered after the account change), which no +/// single caller can do on its own; it is left open deliberately. pub async fn resolve_on_behalf_of( on_behalf_of_email: Option<&str>, on_behalf_of: Option<&str>, @@ -1867,11 +1883,9 @@ pub async fn on_behalf_of_from_permissioned_as( let Some(permissioned_as) = permissioned_as else { return Ok(None); }; - // Uncached: the address is copied onto the job row, where it stays for the life of the run - // and decides the superadmin flag and the instance groups. Nothing evicts the cache across - // processes, so a cached read would keep minting jobs under an address the account no longer - // holds for up to a minute after it moves. - let email = users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db).await?; + // Cached on purpose, up to one notify poll stale: the accepted dispatch case + // `get_email_from_permissioned_as` documents. + let email = users::get_email_from_permissioned_as(permissioned_as, w_id, db).await?; Ok(Some(jobs::OnBehalfOf { email, permissioned_as: permissioned_as.to_string(), diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs index 2656ce5af5..2008e1ce93 100644 --- a/backend/windmill-common/src/queue.rs +++ b/backend/windmill-common/src/queue.rs @@ -15,6 +15,57 @@ pub async fn get_queue_counts(db: &Pool) -> HashMap { .unwrap_or_else(|| HashMap::new()) } +/// Backlog of a single tag: jobs waiting more than 3 seconds past their `scheduled_for`. +pub struct QueueStat { + pub count: u32, + /// How long the job that would be picked up next has already been waiting, in seconds. + pub delay: f64, + /// When that job started waiting (its `scheduled_for`), in epoch seconds. + pub head_since: f64, +} + +/// Same backlog as [`get_queue_counts`], plus the delay of the job at the head of each +/// tag's queue. The head is picked with the same ordering the worker pull uses, so the +/// delay reported is the one a worker is about to observe. +/// +/// Reads the queue of every workspace: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_counts` does. Unlike [`get_queue_counts`], a failed +/// read is an error rather than an empty map, which would read as every backlog draining. +pub async fn get_queue_stats( + db: &Pool, +) -> crate::error::Result> { + // Grouping by (tag, priority) first finds every head in the same single pass as the + // count. A per-tag `ORDER BY ... LIMIT 1` walks `queue_sort_v2`, whose `tag` column comes + // last, through every other tag's backlog queued ahead of it. + let rows = sqlx::query!( + "SELECT tag AS \"tag!\", count AS \"count!\", + EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\", + EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\" + FROM ( + SELECT tag, sum(n)::bigint AS count, + (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head + FROM ( + SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head + FROM v2_job_queue WHERE + scheduled_for <= now() - ('3 seconds')::interval AND running = false + GROUP BY tag, priority + ) g + GROUP BY tag + ) t", + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|x| { + ( + x.tag, + QueueStat { count: x.count as u32, delay: x.delay, head_since: x.head_since }, + ) + }) + .collect()) +} + pub async fn get_queue_running_counts(db: &Pool) -> HashMap { sqlx::query!( "SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE diff --git a/backend/windmill-common/src/queue_metrics.rs b/backend/windmill-common/src/queue_metrics.rs new file mode 100644 index 0000000000..f5affd7346 --- /dev/null +++ b/backend/windmill-common/src/queue_metrics.rs @@ -0,0 +1,482 @@ +//! The queue metrics the monitor samples into `metrics` (`queue_count_{tag}` and +//! `queue_delay_{tag}`), and how a stored series is drawn back. +//! +//! A stored value is a number, held until the next sample, or, for a delay, `{"since": }`: the job at the head of the queue has been waiting since then and was still there +//! when sampled, so the delay climbs one second per second until the next sample. Besides +//! [`QueueSample`], the SQL in [`read_queue_metrics_series`] and in `GET /workers/queue_metrics` +//! decodes both shapes. + +use std::collections::BTreeMap; + +use serde::Serialize; +use sqlx::{Pool, Postgres}; + +pub const QUEUE_COUNT_PREFIX: &str = "queue_count_"; +pub const QUEUE_DELAY_PREFIX: &str = "queue_delay_"; + +/// A backlogged tag whose value has not moved is re-sampled only this often. A longer heartbeat +/// writes fewer rows, but keeps a tag whose drain was never recorded (no server was up when it +/// drained) drawn as backlogged for longer. +pub const QUEUE_METRIC_HEARTBEAT_SECS: f64 = 5.0 * 60.0; + +/// A series silent for longer than this has drained: the sampler stops looking for it, so no +/// closing zero will come, and it is drawn as zero from there. Heartbeats land up to a monitor +/// tick and a sampling slot late, so this must stay well above their real spacing. +pub const QUEUE_METRIC_STALE_SECS: f64 = 3.0 * QUEUE_METRIC_HEARTBEAT_SECS; + +/// Heads that started waiting within this of each other are one wait: jobs queued together +/// leave the head one after another without the delay dropping. +pub const QUEUE_DELAY_SAME_HEAD_SECS: f64 = 1.0; + +/// Slots a series is split into, whatever the window. A slot draws at most four vertices, and a +/// climb one more at each slot boundary it crosses, so a line stays under about 600 points +/// however many rows the window holds. +const QUEUE_METRICS_SERIES_SLOTS: f64 = 120.0; + +/// A stored sample, as it is drawn from the moment it was written until the next one. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum QueueSample { + /// A count, or a delay while the head keeps changing, which hovers. + Held(f64), + /// A delay while the job that started waiting at `since` (epoch seconds) stays at the head. + Climbing { since: f64 }, +} + +impl QueueSample { + pub fn parse(value: &serde_json::Value) -> Option { + match value.get("since") { + Some(since) => since.as_f64().map(|since| Self::Climbing { since }), + None => value.as_f64().map(Self::Held), + } + } + + pub fn to_json(self) -> serde_json::Value { + match self { + Self::Held(value) => serde_json::json!(value), + Self::Climbing { since } => serde_json::json!({ "since": since }), + } + } + + /// Its value at `t`, in epoch seconds. + pub fn value_at(self, t: f64) -> f64 { + match self { + Self::Held(value) => value, + Self::Climbing { since } => t - since, + } + } + + /// When the job at the head of a delay sample written at `at` started waiting. + pub fn head_since(self, at: f64) -> f64 { + match self { + Self::Held(delay) => at - delay, + Self::Climbing { since } => since, + } + } +} + +#[derive(Serialize)] +pub struct QueueMetricsSeries { + /// The window drawn, in epoch milliseconds. + pub from: i64, + pub to: i64, + pub tags: Vec, +} + +#[derive(Serialize)] +pub struct QueueTagSeries { + pub tag: String, + /// Vertices `[epoch ms, value]` of a line joined by straight segments. + pub count: Vec<(i64, f64)>, + pub delay: Vec<(i64, f64)>, +} + +/// The queue metrics of the last `window_secs`, each series aggregated per slot by the database +/// and drawn by [`render_series`], so the size is bounded by the number of tags rather than by +/// how many rows they wrote. +/// +/// Reads the metrics of every workspace's tags: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_metrics_series` does. +pub async fn read_queue_metrics_series( + db: &Pool, + window_secs: f64, +) -> crate::error::Result { + let to = sqlx::query_scalar!("SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"") + .fetch_one(db) + .await?; + let from = to - window_secs; + let slot_secs = window_secs / QUEUE_METRICS_SERIES_SLOTS; + + // Slot -1 holds the samples written before the window, of which only the last is used: it + // sets the value in force at the left edge. A series silent for longer than the stale window + // reads as zero, so nothing older can matter. Arrays compare element by element, so + // `max(ARRAY[t, v])` is the slot's latest sample, found without sorting every row. `v` is a + // sample's value when it was written: for a climbing delay, how long its head had waited. + // + // A climb keeps rising until the next sample, so when that sample lands in the same slot + // (the tag drained, or its head moved), the climb's top is higher than any `v`. Looking the + // next sample up for the slot's last climb, rather than ordering every row, keeps the pass a + // plain aggregate; an earlier climb in the same slot still shows up to its last heartbeat. + // `t` round-trips through `to_timestamp` to within a microsecond either way, so both bounds + // carry a millisecond of slack, far less than two distinct samples of a series are apart: + // without it the climbing sample can match itself, or the one at `last` fall outside. + let rows = sqlx::query!( + "WITH slots AS ( + SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak, + (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value, + (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing, + COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp, + max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb + FROM ( + SELECT id, t, + CASE jsonb_typeof(value) + WHEN 'number' THEN value::double precision + WHEN 'object' THEN t - (value->>'since')::double precision + END AS v, + (value->>'since')::double precision AS since, + (jsonb_typeof(value) = 'object')::int::double precision AS climbing, + greatest(floor((t - $1::double precision) / $2::double precision), -1)::int + AS slot + FROM ( + SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t + FROM metrics + WHERE id LIKE 'queue_%' + AND created_at > to_timestamp($1::double precision - $3::double precision) + ) m + ) s + WHERE v IS NOT NULL + GROUP BY id, slot + ) + SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\", + greatest(peak, CASE WHEN last_climb[1] < last THEN ( + SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision + FROM metrics n + WHERE n.id = slots.id AND n.id LIKE 'queue_%' + AND n.created_at > to_timestamp(last_climb[1] + 0.001) + AND n.created_at <= to_timestamp(last + 0.001) + ) - last_climb[2] END) AS \"peak!\", + first_value AS \"first_value!\", last_value AS \"last_value!\", + last_climbing AS \"last_climbing!\", ramp AS \"ramp!\" + FROM slots + ORDER BY id, slot", + from, + slot_secs, + QUEUE_METRIC_STALE_SECS, + QUEUE_DELAY_SAME_HEAD_SECS, + ) + .fetch_all(db) + .await?; + + #[derive(Default)] + struct Stored { + carried: Option, + slots: Vec, + } + // [count, delay] per tag. + let mut stored: BTreeMap = BTreeMap::new(); + for row in rows { + let (series, tag) = if let Some(tag) = row.id.strip_prefix(QUEUE_COUNT_PREFIX) { + (0, tag) + } else if let Some(tag) = row.id.strip_prefix(QUEUE_DELAY_PREFIX) { + (1, tag) + } else { + continue; + }; + let series = &mut stored.entry(tag.to_string()).or_default()[series]; + let slot = MetricSlot { + first: row.first, + last: row.last, + peak: row.peak, + first_value: row.first_value, + last_value: row.last_value, + last_climbing: row.last_climbing, + ramp: row.ramp, + }; + if row.slot < 0 { + series.carried = Some(slot); + } else { + series.slots.push(slot); + } + } + + let tags = stored + .into_iter() + .map(|(tag, [count, delay])| { + let draw = + |s: &Stored| render_series(s.carried.as_ref(), &s.slots, from, to, slot_secs); + QueueTagSeries { count: draw(&count), delay: draw(&delay), tag } + }) + // A tag that drained before the window has nothing to draw in it. + .filter(|s| s.count.iter().chain(&s.delay).any(|(_, v)| *v != 0.0)) + .collect(); + + Ok(QueueMetricsSeries { + from: (from * 1000.0).round() as i64, + to: (to * 1000.0).round() as i64, + tags, + }) +} + +/// The stored samples of one series that fall in one time slot. +#[derive(Debug, Clone, Copy)] +pub struct MetricSlot { + /// When the first and the last sample of the slot were written, in epoch seconds. + pub first: f64, + pub last: f64, + /// The highest value the series drew over the slot, a climb that ends inside it included. + pub peak: f64, + pub first_value: f64, + /// The value of the last sample, which holds (or climbs, for a climbing delay) until the next. + pub last_value: f64, + pub last_climbing: bool, + /// Every sample of the slot climbs from the same head, so the slot is one exact ramp. + pub ramp: bool, +} + +/// Draw a stored series over `[from, to]` (epoch seconds), split into slots of `slot_secs`, as +/// the vertices of a line joined by straight segments, each `(epoch ms, value)`. +/// +/// A sample holds its value, or a climbing delay keeps climbing, until the next sample or until +/// the series has been silent for [`QUEUE_METRIC_STALE_SECS`]. `carried` is the slot before +/// `from`, whose last sample sets the left edge. A slot draws its peak across the span of its +/// samples, so a spike shorter than a slot still shows at full height, unless it is a single +/// climb, drawn exactly. A climb gets a vertex at every slot boundary it crosses: the delay axis +/// is logarithmic, so one straight segment across many slots would misplace it. +pub fn render_series( + carried: Option<&MetricSlot>, + slots: &[MetricSlot], + from: f64, + to: f64, + slot_secs: f64, +) -> Vec<(i64, f64)> { + let mut line = Line { points: vec![], from, slot_secs }; + let mut held = carried + .map(Held::after) + .filter(|h| from - h.at <= QUEUE_METRIC_STALE_SECS); + if let Some(h) = held { + line.push(from, h.value_at(from)); + } + for slot in slots { + let entering = line.advance(&mut held, slot.first); + line.push(slot.first, entering); + if slot.ramp { + line.push(slot.first, slot.first_value); + } else { + line.push(slot.first, slot.peak); + line.push(slot.last, slot.peak); + } + line.push(slot.last, slot.last_value); + held = Some(Held::after(slot)); + } + if !line.points.is_empty() { + let value = line.advance(&mut held, to); + line.push(to, value); + } + line.points +} + +/// The last sample drawn: when it was written, its value then, and whether it climbs from there. +#[derive(Clone, Copy)] +struct Held { + at: f64, + value: f64, + climbing: bool, +} + +impl Held { + fn after(slot: &MetricSlot) -> Self { + Self { at: slot.last, value: slot.last_value, climbing: slot.last_climbing } + } + + fn value_at(self, t: f64) -> f64 { + if self.climbing { + self.value + (t - self.at) + } else { + self.value + } + } +} + +struct Line { + points: Vec<(i64, f64)>, + from: f64, + slot_secs: f64, +} + +impl Line { + /// The value `held` has at `t`, drawing the climb that leads there and, when the series went + /// silent for too long first, its drop to zero, after which it is forgotten. + fn advance(&mut self, held: &mut Option, t: f64) -> f64 { + let Some(h) = *held else { + return 0.0; + }; + let stale_at = h.at + QUEUE_METRIC_STALE_SECS; + if h.climbing { + let end = t.min(stale_at); + let start = h.at.max(self.from); + let mut boundary = self.from + + ((start - self.from) / self.slot_secs).floor() * self.slot_secs + + self.slot_secs; + while boundary < end { + self.push(boundary, h.value_at(boundary)); + boundary += self.slot_secs; + } + } + if t <= stale_at { + return h.value_at(t); + } + self.push(stale_at, h.value_at(stale_at)); + self.push(stale_at, 0.0); + *held = None; + 0.0 + } + + fn push(&mut self, t: f64, value: f64) { + let point = ((t * 1000.0).round() as i64, value); + match self.points.as_mut_slice() { + [.., last] if *last == point => {} + // A horizontal run only needs its two ends. + [.., a, b] if a.1 == value && b.1 == value => b.0 = point.0, + _ => self.points.push(point), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const FROM: f64 = 1_000_000.0; + const TO: f64 = FROM + 3600.0; + const SLOT: f64 = 30.0; + + fn held(first: f64, last: f64, peak: f64, last_value: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak, + first_value: peak, + last_value, + last_climbing: false, + ramp: false, + } + } + + /// A slot whose samples all climb from a head that started waiting 30s before `FROM`. + fn climbing(first: f64, last: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak: last + 30.0, + first_value: first + 30.0, + last_value: last + 30.0, + last_climbing: true, + ramp: true, + } + } + + fn at(secs: f64, value: f64) -> (i64, f64) { + (((FROM + secs) * 1000.0) as i64, value) + } + + #[test] + fn a_value_holds_until_the_next_sample_and_a_drain_drops_where_it_was_written() { + let line = render_series( + None, + &[ + held(60.0, 60.0, 3.0, 3.0), + held(600.0, 600.0, 2.0, 2.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + SLOT, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(600.0, 3.0), + at(600.0, 2.0), + at(900.0, 2.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } + + #[test] + fn a_series_silent_past_the_stale_window_drops_to_zero() { + let line = render_series(None, &[held(60.0, 60.0, 3.0, 3.0)], FROM, TO, SLOT); + let dropped = 60.0 + QUEUE_METRIC_STALE_SECS; + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(dropped, 3.0), + at(dropped, 0.0), + at(3600.0, 0.0) + ] + ); + } + + #[test] + fn a_slot_draws_its_peak_then_continues_from_its_last_sample() { + // Samples at 60 (5), 70 (9), 80 (4) collapsed into one slot. + let line = render_series( + Some(&held(-30.0, -30.0, 2.0, 2.0)), + &[held(60.0, 80.0, 9.0, 4.0)], + FROM, + FROM + 120.0, + SLOT, + ); + assert_eq!( + line, + vec![ + at(0.0, 2.0), + at(60.0, 2.0), + at(60.0, 9.0), + at(80.0, 9.0), + at(80.0, 4.0), + at(120.0, 4.0) + ] + ); + } + + #[test] + fn a_climbing_delay_is_drawn_exactly_up_to_its_drain() { + // 300s slots: one holds two climbing samples, and heartbeats follow until the drain. + let line = render_series( + None, + &[ + climbing(60.0, 120.0), + climbing(360.0, 360.0), + climbing(660.0, 660.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + 300.0, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + // The slot is one climb, not its peak held across it. + at(60.0, 90.0), + at(120.0, 150.0), + // A vertex at each slot boundary the climb crosses. + at(300.0, 330.0), + at(360.0, 390.0), + at(600.0, 630.0), + at(660.0, 690.0), + // Still climbing right up to the closing zero. + at(900.0, 930.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } +} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 31219621e9..628ae2a709 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -91,9 +91,6 @@ async fn prefetch_cached_script_inner( derive_email: bool, ) -> crate::error::Result> { let derived_email = match script.on_behalf_of.as_deref().filter(|_| derive_email) { - // Uncached: the client preserves this pair and sends it back, where the write path - // validates it against an uncached lookup. A cached address would pair a live principal - // with an address the account no longer holds, and the redeploy would be rejected. Some(permissioned_as) => Some( crate::users::get_email_from_permissioned_as_uncached( permissioned_as, @@ -250,23 +247,37 @@ pub async fn get_full_hub_script_by_path( let version = path_iterator .next() .ok_or_else(|| Error::internal_err(format!("expected hub path to have version number")))?; + // A cache entry that cannot be read or parsed counts as a miss rather than an error: + // a truncated write leaves a file that exists but deserializes to nothing, and refetching + // it is always preferable to failing the job push it was read for. let cache_path = format!("{}/{version}", *HUB_CACHE_DIR); - let script; - if tokio::fs::metadata(&cache_path).await.is_err() { - script = get_full_hub_script_by_path_inner(path, http_client, db).await?; - if let Err(e) = crate::worker::write_file( - &HUB_CACHE_DIR, - &version, - &serde_json::to_string(&script).map_err(to_anyhow)?, - ) { - tracing::error!("failed to write hub script {path} to cache: {e}"); - } else { - tracing::info!("wrote hub script {path} to cache"); + let cached = match tokio::fs::read_to_string(&cache_path).await { + Ok(content) => serde_json::from_str::(&content) + .inspect_err(|e| { + tracing::error!("hub script cache at {cache_path} is unparseable, refetching: {e}") + }) + .ok(), + Err(e) => { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::error!("hub script cache at {cache_path} is unreadable, refetching: {e}"); + } + None } - } else { - let cache_content = tokio::fs::read_to_string(cache_path).await?; - script = serde_json::from_str(&cache_content).unwrap(); + }; + if let Some(script) = cached { tracing::info!("read hub script {path} from cache"); + return Ok(script); + } + + let script = get_full_hub_script_by_path_inner(path, http_client, db).await?; + if let Err(e) = crate::worker::write_file( + &HUB_CACHE_DIR, + &version, + &serde_json::to_string(&script).map_err(to_anyhow)?, + ) { + tracing::error!("failed to write hub script {path} to cache: {e}"); + } else { + tracing::info!("wrote hub script {path} to cache"); } Ok(script) } @@ -358,31 +369,38 @@ pub async fn fetch_script_for_update<'a>( .map_err(crate::error::Error::from) } -pub struct ClonedScript { - pub old_script: NewScript, - pub new_hash: i64, -} -// TODO: What if dependency job fails, there is script with NULL in the lock -pub async fn clone_script<'c>( - path: &str, - w_id: &str, +/// Deploys the outcome of a relative-import relock as a new version of `head`, the path's live +/// version that the caller holds `FOR UPDATE`, and archives `head`. A `lock` of `None` records +/// a failed generation: the version carries `lock_error_logs` instead and runs keep resolving +/// to the last version that has a lock. A `modules` of `None` keeps the head's module locks. +/// +/// Writes whatever `head` names and checks nothing: callers are responsible for having +/// established access to its workspace and path, as a dependency job's push already has. +/// +/// `created_at` is stamped when the insert runs, not at transaction start. The row lock on +/// `head` is what orders one relock after another, and with `now()` a transaction that began +/// first but locked second commits a live child older than its archived parent, which every +/// "latest version" read then mis-orders. +pub async fn deploy_relocked_version( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + head: Script, deployment_message: Option, - db: &DB, -) -> crate::error::Result { - let mut tx = db.begin().await?; - let s = if let Some(s) = fetch_script_for_update(path, w_id, &mut *tx).await? { - s - } else { - return Err(crate::error::Error::NotFound(format!( - "Non-archived script with path '{}' not found", - path - ))); - }; + lock: Option<&str>, + modules: Option<&std::collections::HashMap>, + lock_error_logs: Option<&str>, +) -> crate::error::Result { + let s = head; + let w_id = s.workspace_id.as_str(); - let rs = runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut *tx) - .await?; + let rs = + runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut **tx) + .await?; let (debouncing_settings, concurrency_settings) = - runnable_settings::prefetch_cached_tx(&rs, &mut tx).await?; + runnable_settings::prefetch_cached_tx(&rs, &mut *tx).await?; + + // What the row stores is what the hash covers: the new module locks when there are any. + let modules = modules.cloned().or(s.modules); + let modules_json = modules.as_ref().map(serde_json::to_value).transpose()?; let ns = NewScript { path: s.path.clone(), @@ -392,7 +410,7 @@ pub async fn clone_script<'c>( content: s.content, schema: s.schema, is_template: s.is_template, - lock: None, + lock: lock.map(str::to_string), language: s.language, kind: Some(s.kind), tag: s.tag, @@ -424,7 +442,7 @@ pub async fn clone_script<'c>( on_behalf_of: s.on_behalf_of, preserve_on_behalf_of: None, assets: s.assets, - modules: s.modules, + modules, auto_parent: None, labels: s.labels, skip_draft_deletion: None, @@ -433,7 +451,7 @@ pub async fn clone_script<'c>( let new_hash = hash_script(&ns); tracing::debug!( - "cloning script at path {} from '{}' to '{}'", + "deploying relocked version of script at path {} from '{}' to '{}'", s.path, *s.hash, new_hash @@ -446,17 +464,19 @@ pub async fn clone_script<'c>( envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels) + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, \ + lock_error_logs, created_at) SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ - content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, \ + content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, \ envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, \ + $6::text, clock_timestamp() FROM script WHERE hash = $2 AND workspace_id = $3; - ", new_hash, s.hash.0, w_id).execute(&mut *tx).await?; + ", new_hash, s.hash.0, w_id, lock, modules_json, lock_error_logs).execute(&mut **tx).await?; // Archive base. sqlx::query!( @@ -464,9 +484,8 @@ pub async fn clone_script<'c>( *s.hash, w_id ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - tx.commit().await?; - Ok(ClonedScript { old_script: ns, new_hash }) + Ok(new_hash) } diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 4c50f597ff..31dee749e6 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -6,6 +6,12 @@ pub const ALLOW_PRIVATE_MCP_SERVER_URLS_ENV: &str = "ALLOW_PRIVATE_MCP_SERVER_UR pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METADATA_URLS"; +pub const ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV: &str = "ALLOW_PRIVATE_GUEST_JWKS_URLS"; + +/// Lets every git call reach hosts on a private network, whoever it is made for. +/// Without it, [`private_git_host_allowed`] decides. +pub const ALLOW_LOCAL_GIT_REMOTES_ENV: &str = "ALLOW_LOCAL_GIT_REMOTES"; + /// Why a URL failed SSRF validation. /// /// The distinction matters for callers that gate private endpoints behind a @@ -18,6 +24,9 @@ pub enum SsrfValidationError { InvalidUrl(String), /// Scheme is not `http`/`https`. DisallowedScheme(String), + /// The URL uses `http` where `https` is required (guest JWKS). The private-host opt-in + /// also permits `http`, so, unlike the other scheme errors, this one the flag can fix. + HttpsRequired, /// No host in the URL. MissingHost, /// DNS resolution failed for the host. @@ -37,6 +46,9 @@ impl std::fmt::Display for SsrfValidationError { f, "URL scheme '{s}' is not allowed, only http and https are permitted" ), + SsrfValidationError::HttpsRequired => { + write!(f, "URL must use https") + } SsrfValidationError::MissingHost => write!(f, "URL must have a host"), SsrfValidationError::ResolutionFailed { host, source } => { write!(f, "Failed to resolve host '{host}': {source}") @@ -195,6 +207,54 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } +fn allow_local_git_remotes() -> bool { + std::env::var(ALLOW_LOCAL_GIT_REMOTES_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") +} + +/// Who a git call is made for, which decides whether it may reach a host on a +/// private network. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GitRemoteCaller { + /// A workspace admin's request, or Windmill's own work (polling, webhook and + /// token upkeep, the merge request after a deploy), whose errors only admins read. + AdminOrSystem, + /// A request from anyone who is not a workspace admin. + NonAdmin, +} + +/// Whether a git call made for `caller` may reach a host on a private network. +/// +/// The refusal is for non-admins, who may not be able to run code (operators) +/// and would read git's error output back as a probe of the server's network. +/// An admin can run code, which reaches those hosts from a worker already. On a +/// cloud instance, where a workspace admin is anyone who signed up, every caller +/// is refused. +pub fn private_git_host_allowed(caller: GitRemoteCaller) -> bool { + git_host_policy_allows( + caller, + allow_local_git_remotes(), + *crate::worker::CLOUD_HOSTED, + ) +} + +fn git_host_policy_allows(caller: GitRemoteCaller, opted_in: bool, cloud_hosted: bool) -> bool { + opted_in || (caller == GitRemoteCaller::AdminOrSystem && !cloud_hosted) +} + +/// Appended to a refusal of a private git host, naming what would let `caller` +/// through. `None` where nothing an instance administrator sets would help. +pub fn private_git_host_hint(caller: GitRemoteCaller) -> Option { + (caller == GitRemoteCaller::NonAdmin && !*crate::worker::CLOUD_HOSTED).then(|| { + format!( + "Only workspace admins can reach a git server on a private network. To allow \ + every user, set the {ALLOW_LOCAL_GIT_REMOTES_ENV}=true environment variable on \ + the Windmill servers" + ) + }) +} + pub async fn validate_saml_metadata_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -213,6 +273,36 @@ pub async fn validate_saml_metadata_url(url: &str) -> Result Result { + let parsed = + url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; + + let allow_private = std::env::var(ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1"); + + match parsed.scheme() { + "https" => {} + // Plaintext HTTP only under the explicit operator opt-in that also allows private + // hosts (dev/loopback): the JWKS supplies the keys that authenticate guest JWTs, so an + // on-path attacker who could replace an http response could forge accepted tokens. + "http" if allow_private => {} + "http" => return Err(SsrfValidationError::HttpsRequired), + scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())), + } + + let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; + + if allow_private { + return Ok(ValidatedTarget::unpinned(host)); + } + + validate_url_for_ssrf(url).await +} + pub async fn validate_mcp_server_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -594,6 +684,15 @@ mod tests { )); } + #[test] + fn private_git_hosts_are_refused_to_non_admins_and_on_cloud() { + use GitRemoteCaller::{AdminOrSystem, NonAdmin}; + assert!(git_host_policy_allows(AdminOrSystem, false, false)); + assert!(!git_host_policy_allows(NonAdmin, false, false)); + assert!(!git_host_policy_allows(AdminOrSystem, false, true)); + assert!(git_host_policy_allows(NonAdmin, true, true)); + } + #[tokio::test] async fn saml_ssrf_error_message_includes_env_hint_only_for_private_urls() { let private_error = validate_url_for_ssrf("http://127.0.0.1/metadata") diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 9489e20806..6a982f199d 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -445,6 +445,45 @@ pub async fn fetch_draft_only_list_rows( Ok(rows) } +/// Delete the caller's OWN draft at a path with no deployed row, for the DELETE +/// route of a kind whose list synthesizes such rows via +/// `fetch_draft_only_list_rows`. The `NOT EXISTS` leaves a deployed row's draft +/// alone, so a route may call this on its not-found branch whatever the reason +/// for the miss. `Ok(false)` means nothing matched: the caller reports its own error. +/// +/// Takes no permission check and callers must not add one: an email-scoped row +/// belongs to the caller, who can always discard it, as `update_draft`'s +/// own-discard does. Legacy (`email IS NULL`) rows are owned by nobody and keep +/// their write gate, so discarding one stays on the `update_draft` route. +pub async fn delete_draft_only_for_path( + db: &DB, + w_id: &str, + kind: UserDraftItemKind, + path: &str, + email: &str, +) -> Result { + let Some(table) = kind.deployed_table() else { + return Ok(false); + }; + // `table` is from the closed `deployed_table()` enum, never user input. + let sql = format!( + "DELETE FROM draft \ + WHERE workspace_id = $1 AND typ = $2::text::DRAFT_KIND AND path = $3 \ + AND email = $4 \ + AND NOT EXISTS (SELECT 1 FROM {table} t \ + WHERE t.workspace_id = draft.workspace_id AND t.path = draft.path)" + ); + let deleted = sqlx::query(&sql) + .bind(w_id) + .bind(kind.as_str()) + .bind(path) + .bind(email) + .execute(db) + .await? + .rows_affected(); + Ok(deleted > 0) +} + /// The get-by-path draft choreography, shared by every entity's "get by path" /// route. Given the deployed entity as an `Option` (caller maps its own "not /// found" to `None`): diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 925fe32a4a..1a49605e59 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -31,6 +31,29 @@ pub const USERNAME_GROUP_PREFIX: &str = "group-"; /// columns runnables and triggers store one in. pub const PERMISSIONED_AS_MAX_LEN: usize = 55; +/// Whether any account exists for `email`: a `password` row (deactivated ones +/// included, since the sign-in path filters `disabled = false` and a re-enabled +/// account must not read as absent) or a `usr` row in any workspace (what a service +/// account has instead of a password). A guest is someone with none: the single rule +/// that keeps an account holder from ever holding a cheaper guest identity. +/// +/// The address is lowercased before the lookup: accounts are stored lowercased, so a +/// mixed-case address would otherwise miss an existing account and be let through. The +/// comparison stays a plain equality (not `lower(email)`), so it uses the email index. +pub async fn has_any_account<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, + email: &str, +) -> crate::error::Result { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1) + OR EXISTS(SELECT 1 FROM usr WHERE email = $1)", + ) + .bind(email.to_lowercase()) + .fetch_one(executor) + .await + .map_err(|e| crate::error::Error::internal_err(format!("checking account for {email}: {e:#}"))) +} + /// An email-shaped username is its own principal, which is how a superadmin acting without a /// `usr` row is named (`usr.username` is constrained to `[\w-]+`, so a member never is). It is /// decided before the group convention — an address is never a group's username — and one @@ -152,13 +175,19 @@ pub async fn permissioned_as_exists( /// Drop a cached address so a transactional email change is visible immediately. /// -/// The address is derived at dispatch and feeds the instance-superadmin check and -/// `email_to_igroup`, so serving a stale one would run jobs with the wrong authorization -/// for up to the cache TTL. +/// Not the thing that keeps authorization correct — `fetch_authed_from_permissioned_as` +/// re-resolves the address before granting anything. This keeps the cache from serving an +/// address that is merely wrong for the TTL, on reads and on what is shown. pub fn invalidate_email_cache(workspace_id: &str, username: &str) { EMAIL_CACHE.remove(&(workspace_id.to_string(), username.to_string())); } +/// Drop this name's entry in every workspace, for the changes that know the name but not the +/// workspace: a superadmin resolves through `password`, whose row names no workspace of its own. +pub fn invalidate_email_cache_for_username(username: &str) { + EMAIL_CACHE.retain(|(_workspace_id, cached_username), _| cached_username != username); +} + /// Inverse of [`get_email_from_permissioned_as`]: the principal an on-behalf-of email /// names in this workspace, for callers that supply the email alone. /// @@ -171,6 +200,14 @@ pub fn invalidate_email_cache(workspace_id: &str, username: &str) { /// not a superadmin's, or a group that no longer exists. Callers then leave the identity /// unrecorded rather than storing a principal that cannot authenticate. /// +/// Known, accepted consequence of a real account winning the synthetic `group-*@windmill.dev` +/// namespace: a group identity sent as its address alone, as a "keep target identity" workspace +/// deploy sends it for scripts, flows and apps, comes back as the account holding that address +/// when one exists, not as `g/*`. Such an account takes an admin to exist: a superadmin or an +/// admin-configured identity provider to create it (the public OAuth providers only assert a +/// `@windmill.dev` address to that domain's owner) and an admin of the target workspace to admit +/// it, so no member can steer a group's runnables to themselves this way. +/// /// Reads through the non-RLS pool and authorizes nothing: callers must already be authorized /// for `workspace_id`. pub async fn permissioned_as_from_email( @@ -219,6 +256,33 @@ pub async fn permissioned_as_from_email( /// - "u/{username}" → resolve via [`resolve_username_to_email`] (cached) /// - "g/{group}" → "group-{group}@windmill.dev" /// - raw email → return as-is +/// +/// `notify_user_email_change` evicts the key on every process for each change that can move it, +/// at that process's next notify-event poll (`LISTEN_NEW_EVENTS_INTERVAL_SEC`, 10s by default), +/// so a hit can still be the old address for up to one poll. The TTL caps it if an eviction is +/// ever missed. +/// +/// Which of the two to use is a question of how long a wrong answer lives, not of whether it is +/// stored — both of these get stored and read back. A config row (an app policy, a schedule, a +/// runnable) is the authority for every run that follows it, so a stale address there is +/// permanent and invisible: those use [`get_email_from_permissioned_as_uncached`]. Job dispatch +/// also stores its answer, and the worker reads it back to build that run's authed, but it +/// governs one job and dies with it, so it stays here. +/// +/// The job's own authorization does not trust the address as given: +/// `fetch_authed_from_permissioned_as` re-resolves it from the principal's live binding, and that +/// corrected address is what the job row and its token carry. Route an address into an `Authed`, +/// a job row or a token without going through that function, and this cache stops being safe to +/// read at dispatch. +/// +/// What reads the dispatch address before that re-resolution (the quota and superadmin-exemption +/// checks at the top of `push_inner`, a flow step's tag check) or when the principal has no live +/// binding can act on the old address for up to one poll after a username reuse, an email change +/// or a superadmin change. That window is accepted as the cost of keeping dispatch off the +/// database; a consumer that cannot tolerate it must re-resolve first. +/// +/// Reads through the non-RLS pool and authorizes nothing — callers must already be authorized +/// for `workspace_id`. pub async fn get_email_from_permissioned_as<'c>( permissioned_as: &str, workspace_id: &str, @@ -227,13 +291,21 @@ pub async fn get_email_from_permissioned_as<'c>( get_email_from_permissioned_as_inner(permissioned_as, workspace_id, db, true).await } -/// [`get_email_from_permissioned_as`] without the address cache. Nothing evicts that cache -/// across processes, so for a minute after an email change it still serves the old address — -/// fine where the address only labels something on screen, wrong where it decides whether a -/// write is accepted or is copied onto a job row that outlives the window. +/// [`get_email_from_permissioned_as`] for a value about to be **persisted**. /// -/// Reads through the non-RLS pool and authorizes nothing, like the cached one: callers must -/// already be authorized for `workspace_id`. +/// The eviction is delivered by the `notify_event` poller, not synchronously, so for a few +/// seconds after a change a replica can still serve the old address. In a config row that is +/// permanent: the row outlives the eviction, every later run trusts it, and nothing re-derives +/// it, so a principal and an address that name different accounts stay that way. +/// +/// Use this for three cases, all of which end in a stored pair: +/// - writing the address into a row; +/// - the lookup that validates a pair before it is stored; +/// - **reads whose result the client sends back** — a script or a workspace export hands over a +/// principal and address together, and a redeploy validates that pair against a fresh +/// resolution, so a stale one comes back as a rejected deploy rather than a stale display. +/// +/// See [`get_email_from_permissioned_as`] for the dispatch case that deliberately does not. pub async fn get_email_from_permissioned_as_uncached<'c>( permissioned_as: &str, workspace_id: &str, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 84284759c6..ee8a389bb7 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -392,6 +392,7 @@ lazy_static::lazy_static! { pip_local_dependencies: Default::default(), env_vars: Default::default(), native_mode: false, + object_store_cache_config: Default::default(), }); pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); @@ -2343,6 +2344,7 @@ pub async fn load_worker_config( .or_else(|| load_additional_python_paths_from_env()), env_vars: resolved_env_vars, native_mode, + object_store_cache_config: config.object_store_cache_config, }) } @@ -2432,6 +2434,7 @@ pub struct WorkerConfigOpt { pub env_vars_static: Option>, pub env_vars_allowlist: Option>, pub native_mode: Option, + pub object_store_cache_config: Option, } impl Default for WorkerConfigOpt { @@ -2450,6 +2453,7 @@ impl Default for WorkerConfigOpt { env_vars_static: Default::default(), env_vars_allowlist: Default::default(), native_mode: Default::default(), + object_store_cache_config: Default::default(), } } } @@ -2468,12 +2472,18 @@ pub struct WorkerConfig { pub pip_local_dependencies: Option>, pub env_vars: HashMap, pub native_mode: bool, + /// Object store this group's dependency cache uses instead of the instance one, as stored + /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate + /// that parses it, and comparing the raw value is what tells a reload the store changed. + pub object_store_cache_config: Option, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode) + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, object_store_cache_config: {} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, + // holds bucket credentials + self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None")) } } @@ -2502,6 +2512,42 @@ pub fn split_python_requirements>(requirements: T) -> Vec .collect() } +/// Byte offset of the comment marker, per pip's rule: a `#` at line start or preceded by +/// whitespace. A `#` elsewhere belongs to the requirement (`pkg @ https://h/p.whl#sha256=…`). +fn requirement_comment_start(line: &str) -> Option { + line.char_indices() + .find(|(i, c)| *c == '#' && (*i == 0 || line[..*i].ends_with(char::is_whitespace))) + .map(|(i, _)| i) +} + +/// The installable requirement carried by one lockfile line, or `None` for a comment, a +/// `-r`/`-e`/`--flag` directive, or a blank. +/// +/// Windmill installs a lockfile one entry at a time as a `uv pip install` argument, so +/// requirements-file syntax a file-level parser would absorb is an unparseable package name +/// here and has to be stripped first. +pub fn requirement_from_lockfile_line(line: &str) -> Option<&str> { + let requirement = match requirement_comment_start(line) { + Some(i) => &line[..i], + None => line, + } + .trim() + // Continuations are stripped, not joined: right for `--generate-hashes` locks, whose + // continued lines are `--hash=` flags this function drops, but a lock continuing onto a + // marker or extra would lose it. + .trim_end_matches('\\') + .trim_end(); + + (!requirement.is_empty() && !requirement.starts_with('-')).then_some(requirement) +} + +/// Whether a lockfile line continues onto the next one. The continued lines reach the +/// installer as entries of their own rather than being joined, so a caller that cares what +/// they carried — `--hash=` pins, for a `--generate-hashes` lock — has to say so itself. +pub fn lockfile_line_has_continuation(line: &str) -> bool { + line.trim_end().ends_with('\\') +} + #[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] #[repr(u32)] pub enum PyVAlias { @@ -2620,6 +2666,52 @@ mod tests { ids.iter().map(|s| s.to_string()).collect() } + /// Fixtures are verbatim `uv pip compile` output (uv 0.11.28): split and inline + /// annotation styles, and `--generate-hashes`. + #[test] + fn test_requirement_from_lockfile_line() { + assert_eq!(requirement_from_lockfile_line(" # via httpx"), None); + assert_eq!(requirement_from_lockfile_line(" # via"), None); + assert_eq!(requirement_from_lockfile_line(" # anyio"), None); + assert_eq!( + requirement_from_lockfile_line(" # via -r .tmp/requirements.in"), + None + ); + assert_eq!( + requirement_from_lockfile_line("anyio==4.15.1 \\"), + Some("anyio==4.15.1") + ); + assert_eq!( + requirement_from_lockfile_line( + " --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7 \\" + ), + None + ); + assert_eq!(requirement_from_lockfile_line("# py: 3.11"), None); + assert_eq!(requirement_from_lockfile_line("-r other.txt"), None); + assert_eq!( + requirement_from_lockfile_line("--index-url https://x"), + None + ); + assert_eq!(requirement_from_lockfile_line(" "), None); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0"), + Some("httpx==0.27.0") + ); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0 # via -r requirements.in"), + Some("httpx==0.27.0") + ); + // A `#` not preceded by whitespace is part of the requirement, not a comment. + assert_eq!( + requirement_from_lockfile_line("wmill @ https://h/wmill.whl#sha256=abc"), + Some("wmill @ https://h/wmill.whl#sha256=abc") + ); + + assert!(lockfile_line_has_continuation("anyio==4.15.1 \\")); + assert!(!lockfile_line_has_continuation("anyio==4.15.1")); + } + #[test] fn test_parse_job_oom_score_adj() { assert_eq!(parse_job_oom_score_adj(Some("300")), 300); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 697fc69e44..ad040c5905 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -71,6 +71,7 @@ bitflags::bitflags! { const RESTRICT_DEPLOY_TO_DEPLOYERS = 1 << 2; const RESTRICT_ANONYMOUS_APP_DEPLOYMENT = 1 << 3; const RESTRICT_PUBLIC_RUN_SHARING = 1 << 4; + const RESTRICT_GUEST_APP_DEPLOYMENT = 1 << 5; } } @@ -83,6 +84,7 @@ pub enum ProtectionRuleKind { RestrictDeployToDeployers, RestrictAnonymousAppDeployment, RestrictPublicRunSharing, + RestrictGuestAppDeployment, } impl ProtectionRuleKind { @@ -103,6 +105,9 @@ impl ProtectionRuleKind { ProtectionRuleKind::RestrictPublicRunSharing => { ProtectionRules::RESTRICT_PUBLIC_RUN_SHARING } + ProtectionRuleKind::RestrictGuestAppDeployment => { + ProtectionRules::RESTRICT_GUEST_APP_DEPLOYMENT + } } } @@ -121,6 +126,9 @@ impl ProtectionRuleKind { ProtectionRuleKind::RestrictPublicRunSharing => { "Sharing a run publicly (readable without login) is restricted in this workspace" } + ProtectionRuleKind::RestrictGuestAppDeployment => { + "Opening an app to guests (anyone who can sign in) is restricted in this workspace" + } } } } @@ -175,7 +183,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28958/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -183,7 +191,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28930/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28957/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -337,13 +345,14 @@ pub struct GitRepositorySettings { #[serde(default, skip_serializing_if = "Option::is_none")] pub auto_pull: Option, /// Open a PR when a deploy pushes a `wm_deploy/**` branch of this promotion - /// repo (app-backed only; runs from the deploy callback so it works without + /// repo (needs a credential the server holds — a GitHub App installation or + /// a checked GitLab token; runs from the deploy callback so it works without /// inbound webhooks). Off by default so upgrades don't change behavior. #[serde(default, skip_serializing_if = "is_false")] pub promotion_open_prs: bool, /// Parent-level: open a PR when a fork of this workspace deploys to its - /// `wm-fork/**` branch (app-backed only; the fork's deploy callback reads - /// this from the parent). Off by default. + /// `wm-fork/**` branch (needs a credential the server holds; the fork's + /// deploy callback reads this from the parent). Off by default. #[serde(default, skip_serializing_if = "is_false")] pub fork_open_prs: bool, /// Server-owned: the last failure opening a PR for a deploy branch of this @@ -352,6 +361,10 @@ pub struct GitRepositorySettings { /// successful PR; never accepted from clients. #[serde(default, skip_serializing_if = "Option::is_none")] pub open_pr_error: Option, + /// Server-owned: what the repo's credential says about its own expiry and + /// scopes. Written by the credential check, never accepted from clients. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential: Option, } impl GitRepositorySettings { @@ -397,6 +410,44 @@ pub enum AutoPullMode { Polling, } +/// Host whose credential lifecycle Windmill can manage from the repo URL. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum GitCredentialProvider { + Gitlab, +} + +/// What the repo's own credential says about itself, refreshed by asking the +/// host. Server-owned: written by the credential check, never accepted from a +/// client. +/// +/// Absent means the check has not run or the repo carries no credential we can +/// introspect (a GitHub App repo mints tokens per call and has nothing to expire). +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct GitCredentialStatus { + pub provider: GitCredentialProvider, + /// Changes on every rotation, so it identifies the current token, not the + /// credential's whole history. + #[serde(skip_serializing_if = "Option::is_none")] + pub token_id: Option, + /// `None` is a non-expiring token, which only self-managed GitLab can issue + /// (and only for a service account). It means no warning and no rotation. + #[serde(skip_serializing_if = "Option::is_none")] + pub expires_at: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scopes: Vec, + /// Whether *this workspace* renews the credential. That needs a scope which + /// permits it (`api` or `self_rotate`) and a credential this workspace holds: + /// a token carried in the repository URL is the operator's to manage, and one + /// resolved from an ancestor is the ancestor's, so neither is renewed here. + pub rotatable: bool, + /// Unix timestamp (seconds) of the last check. + pub checked_at: i64, + /// Why the last check or rotation failed, cleared by the next success. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + /// Outcome of the most recent auto-pull attempt, surfaced in the UI. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AutoPullStatus { @@ -508,9 +559,10 @@ impl AutoPullSettings { /// Whether a freshly observed `(git_ref, head_sha)` warrants enqueuing a pull. /// /// A trigger (poll or webhook) is only a hint: we pull when auto-pull is - /// enabled and the observed head differs from the last sha we synced for - /// that ref. Re-observing the same head (e.g. a redundant poll, or the - /// commit our own deploy callback just pushed back) is a no-op. + /// enabled and the observed head differs from the last sha we pulled for + /// that ref. Re-observing the same head (a redundant poll) is a no-op. A + /// commit our own deploy pushed is not: pushes never write here, so the pull + /// it triggers picks up anything pushed under it. pub fn should_pull(&self, git_ref: &str, head_sha: &str) -> bool { self.enabled && self.last_synced_sha.get(git_ref).map(String::as_str) != Some(head_sha) } @@ -767,6 +819,223 @@ pub struct BillableSeats { pub seats: i64, } +/// Guests are free up to `FREE_GUESTS_PER_WINDOW` distinct emails over the trailing +/// `GUEST_WINDOW_DAYS`. Past that, an Enterprise plan meters them, `GUESTS_PER_SEAT` +/// guests to one seat, while every other plan and build stops admitting new emails. +pub const GUEST_WINDOW_DAYS: i32 = 30; +pub const FREE_GUESTS_PER_WINDOW: i64 = 100; +pub const GUESTS_PER_SEAT: i64 = 4; + +/// Whether guests past the allowance are metered (Enterprise plan) rather than refused. +/// A build without `enterprise` has no plan and is capped, like a Pro key. +pub async fn guests_are_metered() -> bool { + #[cfg(feature = "enterprise")] + { + matches!( + crate::ee_oss::get_license_plan().await, + crate::ee_oss::LicensePlan::Enterprise + ) + } + #[cfg(not(feature = "enterprise"))] + { + false + } +} + +/// Seats the guests past the free allowance consume: `ceil(billable / GUESTS_PER_SEAT)`. +pub fn guest_seats(distinct_guests: i64) -> i64 { + let billable = (distinct_guests - FREE_GUESTS_PER_WINDOW).max(0); + (billable + GUESTS_PER_SEAT - 1) / GUESTS_PER_SEAT +} + +/// Distinct guest emails over the trailing window, today included. +pub async fn guest_count_in_window<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, +) -> Result { + sqlx::query_scalar( + "SELECT COUNT(DISTINCT email) FROM guest_activity WHERE day > CURRENT_DATE - $1", + ) + .bind(GUEST_WINDOW_DAYS) + .fetch_one(executor) + .await + .map_err(|e| Error::internal_err(format!("counting guests: {e:#}"))) +} + +/// The instance's standing against the guest allowance, as every surface reports it. +#[derive(Clone, Debug, Serialize)] +pub struct GuestUsage { + /// Whether this deployment can admit guests at all ([`instance_supports_guests`]). + /// Off, every other field is moot and no switch below can turn guests on. + pub available: bool, + /// The superadmin switch (`GUEST_ACCESS_DISABLED_SETTING`), which every workspace + /// switch sits under. Reported as stored, so a superadmin sees what they set even + /// where `available` overrules it. + pub instance_enabled: bool, + /// Distinct guest emails over the trailing `window_days`. + pub guest_count: i64, + pub window_days: i32, + pub free_allowance: i64, + /// Enterprise plan: guests past the allowance take `guest_seats`. Otherwise no new + /// email is admitted past it. + pub metered: bool, + pub billable_guests: i64, + pub guest_seats: i64, +} + +/// What a caller is told when it asks for guests on a deployment that cannot have them. +pub const GUESTS_UNAVAILABLE_MESSAGE: &str = + "Guest access is not available on Windmill Cloud. It requires a self-hosted instance \ + or a dedicated Windmill Cloud deployment."; + +/// Whether guests can exist on this deployment at all. They cannot on the shared cloud: +/// a guest is an identity Windmill itself never vouched for, admitted on the say-so of +/// whoever runs the instance, which is not a call a multi-tenant deployment can make for +/// its tenants. Folded into every guest gate below, so a workspace switch or an app +/// policy left saying `guest` is inert rather than honored. +pub fn instance_supports_guests() -> bool { + !*crate::worker::CLOUD_HOSTED +} + +/// [`instance_supports_guests`] as an error, for the writes that would otherwise store a +/// setting that can never take effect. +pub fn require_guest_support() -> Result<()> { + if instance_supports_guests() { + Ok(()) + } else { + Err(Error::BadRequest(GUESTS_UNAVAILABLE_MESSAGE.to_string())) + } +} + +/// SQL for the superadmin switch alone, absent meaning on. The setting is read as text +/// before the cast so `true` and `"true"` both count. +fn instance_switch_sql() -> String { + format!( + "NOT COALESCE((SELECT (value #>> '{{}}')::boolean FROM global_settings \ + WHERE name = '{}'), false)", + crate::global_settings::GUEST_ACCESS_DISABLED_SETTING + ) +} + +/// SQL for "the instance admits guests": the superadmin switch, under +/// [`instance_supports_guests`]. +fn instance_admits_guests_sql() -> String { + if !instance_supports_guests() { + return "false".to_string(); + } + instance_switch_sql() +} + +pub async fn guest_usage(db: &crate::DB) -> Result { + let instance_switch = instance_switch_sql(); + let instance_enabled: bool = sqlx::query_scalar(&format!("SELECT {instance_switch}")) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("reading the instance guest switch: {e:#}")))?; + let guest_count = guest_count_in_window(db).await?; + let metered = guests_are_metered().await; + let billable_guests = if metered { + (guest_count - FREE_GUESTS_PER_WINDOW).max(0) + } else { + 0 + }; + Ok(GuestUsage { + available: instance_supports_guests(), + instance_enabled, + guest_count, + window_days: GUEST_WINDOW_DAYS, + free_allowance: FREE_GUESTS_PER_WINDOW, + metered, + billable_guests, + guest_seats: if metered { guest_seats(guest_count) } else { 0 }, + }) +} + +/// Whether `email` may be admitted as a guest right now. Checked once, where a session +/// is minted: a returning guest (already in the window) is always let back in, so the +/// cap only ever refuses a stranger, and a metered instance refuses nobody. +/// +/// Must run inside the transaction that then records the guest in `guest_activity`: +/// it takes a transaction-scoped lock so concurrent strangers count each other, and the +/// lock is what keeps the cap exact rather than approximate. +pub async fn guest_admission(conn: &mut sqlx::PgConnection, email: &str) -> Result<()> { + if guests_are_metered().await { + return Ok(()); + } + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('guest_allowance'))") + .execute(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("locking the guest allowance: {e:#}")))?; + let (in_window, count): (bool, i64) = sqlx::query_as( + "SELECT + EXISTS(SELECT 1 FROM guest_activity WHERE email = $1 AND day > CURRENT_DATE - $2), + (SELECT COUNT(DISTINCT email) FROM guest_activity WHERE day > CURRENT_DATE - $2)", + ) + .bind(email) + .bind(GUEST_WINDOW_DAYS) + .fetch_one(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("checking the guest allowance: {e:#}")))?; + if in_window || count < FREE_GUESTS_PER_WINDOW { + return Ok(()); + } + Err(Error::PermissionDenied(format!( + "This instance has reached its limit of {FREE_GUESTS_PER_WINDOW} guests over \ + {GUEST_WINDOW_DAYS} days. Guest sign-in beyond that needs an Enterprise license." + ))) +} + +/// Whether a guest session for `email` in `w_id` still stands: the instance and the +/// workspace admit guests, and the email still has no account. Read at the auth door on +/// every guest request, so turning either switch off, or an account provisioned after +/// the mint (or racing it), ends the session on its next request. +pub async fn guest_session_stands(db: &crate::DB, w_id: &str, email: &str) -> Result { + let instance_admits = instance_admits_guests_sql(); + let stands: Option = sqlx::query_scalar(&format!( + "SELECT guest_access_enabled + AND {instance_admits} + AND NOT EXISTS(SELECT 1 FROM password WHERE email = $2) + AND NOT EXISTS(SELECT 1 FROM usr WHERE email = $2) + FROM workspace_settings WHERE workspace_id = $1" + )) + .bind(w_id) + .bind(email) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("checking the guest session of {email}: {e:#}")))?; + Ok(stands.unwrap_or(false)) +} + +/// Every switch at once: the instance's, the workspace's, and `app_path` being in +/// `guest` execution mode. The single answer to "may a guest session be minted for this +/// app", used by the mint itself and by the sign-in branch that decides whether to call +/// it. A missing app or a policy with no stated mode reads as "no". The allowance is +/// `guest_admission`. +pub async fn guest_app_admits<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, + w_id: &str, + app_path: &str, +) -> Result { + // The mint refuses a path it cannot scope, so discovery must not advertise one. + if !crate::auth::is_scope_literal_path(app_path) { + return Ok(false); + } + let instance_admits = instance_admits_guests_sql(); + let admits: Option = sqlx::query_scalar(&format!( + "SELECT COALESCE(ws.guest_access_enabled AND app.policy->>'execution_mode' = 'guest', false) + AND {instance_admits} + FROM app JOIN workspace_settings ws ON ws.workspace_id = app.workspace_id + WHERE app.workspace_id = $1 AND app.path = $2" + )) + .bind(w_id) + .bind(app_path) + .fetch_optional(executor) + .await + .map_err(|e| { + Error::internal_err(format!("checking guest access to {w_id}/{app_path}: {e:#}")) + })?; + Ok(admits.unwrap_or(false)) +} + /// Billable members of `w_id` and the seats they cost, as `ceil(developers + operators/2)`. Service /// accounts cannot log in and do not take a seat; a disabled member is not billed either. /// @@ -2804,3 +3073,17 @@ pub async fn dbt_warehouse_resource( .map(|t| t.to_string()); Ok((path, target)) } + +#[cfg(test)] +mod guest_allowance_tests { + use super::*; + + #[test] + fn guest_seats_round_up_past_the_allowance() { + assert_eq!(guest_seats(0), 0); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW), 0); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW + 1), 1); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW + GUESTS_PER_SEAT), 1); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW + GUESTS_PER_SEAT + 1), 2); + } +} diff --git a/backend/windmill-common/tests/dbt_graph_storage.rs b/backend/windmill-common/tests/dbt_graph_storage.rs index 7b4971312a..3226d27e9e 100644 --- a/backend/windmill-common/tests/dbt_graph_storage.rs +++ b/backend/windmill-common/tests/dbt_graph_storage.rs @@ -7,9 +7,10 @@ use sqlx::{Pool, Postgres}; use windmill_common::dbt_manifest::{ - clear_dbt_editor_graphs, clear_dbt_manifest_version, prune_dbt_run_graphs, - replace_dbt_editor_graph, replace_dbt_manifest, IngestedManifest, IngestedNode, - DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, + clear_dbt_editor_graphs, clear_dbt_manifest_version, clear_dbt_script_state, + clear_dbt_script_state_if_path_retired, move_dbt_script_state, prune_dbt_run_graphs, + replace_dbt_editor_graph, replace_dbt_manifest, IngestedColumnEdge, IngestedManifest, + IngestedNode, DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, }; const WS: &str = "test-workspace"; @@ -52,10 +53,36 @@ fn manifest(names: &[&str]) -> IngestedManifest { .windows(2) .map(|w| (format!("model.p.{}", w[0]), format!("model.p.{}", w[1]))) .collect(), + // Same reason: a project that opted into the analysis pass has these, and + // a fixture without them leaves every column-edge insert and sweep in + // this file unexecuted. + column_edges: names + .windows(2) + .map(|w| IngestedColumnEdge { + parent_unique_id: format!("model.p.{}", w[0]), + parent_column: w[0].to_string(), + child_unique_id: format!("model.p.{}", w[1]), + child_column: w[1].to_string(), + lineage_kind: "copy".to_string(), + }) + .collect(), ..Default::default() } } +/// Column edges of one version, so the sweeps can be shown to reach them. +async fn column_edges_for(db: &Pool, hash: i64) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + WS, + hash + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} + /// Edges for one version, so a test can assert the batched insert ran at all. async fn edges_for(db: &Pool, hash: i64) -> i64 { sqlx::query_scalar!( @@ -135,6 +162,33 @@ async fn an_identical_run_stores_no_snapshot(db: Pool) { assert_eq!(markers(&db, 1).await, 1, "and leaves no marker of its own"); } +/// A column that is projected AND used as a predicate for the same output column +/// has both a `copy` edge and a `scan` one. They are two facts, and the digest +/// counts both — so the uniqueness key has to carry `lineage_kind`, or the +/// second is dropped by `ON CONFLICT DO NOTHING` while the digest still claims +/// it was stored. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn both_kinds_of_one_column_pair_are_stored(db: Pool) { + deploy_script(&db, 1).await; + let pair = |kind: &str| IngestedColumnEdge { + parent_unique_id: "model.p.a".to_string(), + parent_column: "id".to_string(), + child_unique_id: "model.p.b".to_string(), + child_column: "id".to_string(), + lineage_kind: kind.to_string(), + }; + let mut m = manifest(&["a", "b"]); + m.column_edges = vec![pair("copy"), pair("scan")]; + + let mut tx = db.begin().await.unwrap(); + replace_dbt_manifest(&mut tx, WS, PATH, 1, None, &m, "root") + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!(column_edges_for(&db, 1).await, 2, "both kinds survive"); +} + /// A run whose model set differs keeps its own, and the version's is untouched: /// this is what lets an older run page render the project that run built. #[sqlx::test(migrations = "../migrations", fixtures("base"))] @@ -235,7 +289,11 @@ async fn clearing_one_version_leaves_the_others(db: Pool) { // this is where two versions coexist: it pins the batched edge insert // against a real database as well as the version scoping. assert_eq!(edges_for(&db, 1).await, 0, "the cleared version's edges go"); - assert_eq!(edges_for(&db, 2).await, 1, "the other version keeps its own"); + assert_eq!( + edges_for(&db, 2).await, + 1, + "the other version keeps its own" + ); } /// The routes that hard-delete a path clear no graph rows: they delete the @@ -275,6 +333,8 @@ async fn deleting_the_script_cascades_to_every_sidecar(db: Pool) { assert_eq!(nodes_for(&db, 2, DEPLOYED_GRAPH).await, 0); assert_eq!(edges_for(&db, 1).await, 0); assert_eq!(edges_for(&db, 2).await, 0); + assert_eq!(column_edges_for(&db, 1).await, 0); + assert_eq!(column_edges_for(&db, 2).await, 0); assert_eq!(markers_for_path(&db).await, 0); } @@ -315,7 +375,7 @@ async fn the_sweep_takes_old_snapshots_and_spares_the_version(db: Pool tx.commit().await.unwrap(); // Age one snapshot past the window, rows and marker together. - for t in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for t in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "UPDATE {t} SET ingested_at = now() - interval '400 days' WHERE job_id = $1" )) @@ -363,7 +423,11 @@ async fn only_the_newest_deploys_keep_their_graph(db: Pool) { // The newest is always among them: losing the live version's graph would // empty the page of every run of it. assert_eq!(nodes_for(&db, over, DEPLOYED_GRAPH).await, 1); - assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0, "the oldest is reclaimed"); + assert_eq!( + nodes_for(&db, 1, DEPLOYED_GRAPH).await, + 0, + "the oldest is reclaimed" + ); } /// The third provenance: a `parse` of the EDITOR's buffer, which names no @@ -480,17 +544,27 @@ async fn a_version_clear_spares_editor_graphs_and_a_path_clear_does_not(db: Pool replace_dbt_editor_graph(&mut tx, WS, PATH, job, ME, &manifest(&["a"]), "root") .await .unwrap(); - clear_dbt_manifest_version(&mut tx, WS, PATH, 1).await.unwrap(); + clear_dbt_manifest_version(&mut tx, WS, PATH, 1) + .await + .unwrap(); tx.commit().await.unwrap(); assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0); - assert_eq!(editor_nodes(&db, job).await, 1, "the buffer's graph survives"); + assert_eq!( + editor_nodes(&db, job).await, + 1, + "the buffer's graph survives" + ); let mut tx = db.begin().await.unwrap(); clear_dbt_editor_graphs(&mut tx, WS, PATH).await.unwrap(); tx.commit().await.unwrap(); - assert_eq!(editor_nodes(&db, job).await, 0, "retiring the path takes it"); + assert_eq!( + editor_nodes(&db, job).await, + 0, + "retiring the path takes it" + ); } /// A preview names its own PATH and needs only `jobs:run`, so a bound over the @@ -552,3 +626,157 @@ async fn editor_markers(db: &Pool) -> i64 { .unwrap() .unwrap_or(0) } + +/// A deferral resolves a `ref()` through the manifest of the last successful run +/// at this path, so that state has to follow the script the way the retry state +/// does: a rename must not strand it, and a path no live dbt version occupies +/// must not hand its manifest to whatever is created there next. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn environment_state_follows_the_script(db: Pool) { + const MOVED: &str = "f/test/renamed"; + deploy_script(&db, 1).await; + publish_environment_state(&db, PATH).await; + + let mut tx = db.begin().await.unwrap(); + move_dbt_script_state(&mut tx, WS, PATH, MOVED) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, PATH).await, 0); + assert_eq!(environment_states(&db, MOVED).await, 1); + + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state(&mut tx, WS, MOVED).await.unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, MOVED).await, 0); +} + +/// Archiving or deleting ONE version must not take the path's state with it — +/// the live version's next deferral still needs it — while the last one leaving +/// must, or a script later created at that path inherits the previous project's +/// manifest. The condition is a `NOT EXISTS` in raw SQL, so both directions are +/// pinned against a real database. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn state_goes_only_once_no_live_version_is_left(db: Pool) { + deploy_script(&db, 1).await; + deploy_script(&db, 2).await; + publish_environment_state(&db, PATH).await; + + retire(&db, 1).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 1, + "another version is still live here" + ); + + retire(&db, 2).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 0, + "the last one leaving takes it" + ); +} + +async fn retire(db: &Pool, hash: i64) { + sqlx::query!( + "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + WS, + hash + ) + .execute(db) + .await + .unwrap(); +} + +/// The worker publishes under a guard naming the version that ran, and the whole +/// point of it is a job that finishes late: its script can be renamed away and an +/// unrelated one created at the same path while it runs, and that project must +/// not inherit this one's manifest as its deferral state. Enforced in raw SQL, +/// where a refactor can drop a predicate with no type error, so it is pinned +/// against a real database — the same shape `dbt_state::publish` issues. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_late_job_cannot_publish_for_a_path_it_no_longer_owns(db: Pool) { + deploy_script(&db, 1).await; + assert_eq!(guarded_publish(&db, PATH, 1).await, 1, "its own version"); + assert_eq!( + guarded_publish(&db, PATH, 2).await, + 0, + "a version that never lived here" + ); + + // The script is gone from this path and another one takes it. + sqlx::query!( + "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + WS + ) + .execute(&db) + .await + .unwrap(); + deploy_script(&db, 3).await; + assert_eq!( + guarded_publish(&db, PATH, 1).await, + 0, + "the late job's version does not own this path any more" + ); +} + +/// The predicate `dbt_state::publish` locks the script row on, reduced to what it +/// decides. Keep the two in step — this file cannot call `publish` itself, which +/// is `pub(crate)` in `windmill-worker`. +async fn guarded_publish(db: &Pool, path: &str, ran: i64) -> u64 { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text + WHERE EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false + AND language = 'dbt' + AND (hash = $4 OR $4 = ANY(parent_hashes))) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + WS, + path, + uuid::Uuid::from_u128(9), + ran, + ) + .execute(db) + .await + .unwrap() + .rows_affected() +} + +async fn publish_environment_state(db: &Pool, path: &str) { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + WS, + path, + uuid::Uuid::from_u128(9), + ) + .execute(db) + .await + .unwrap(); +} + +async fn environment_states(db: &Pool, path: &str) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} diff --git a/backend/windmill-common/tests/dbt_producer_rules.rs b/backend/windmill-common/tests/dbt_producer_rules.rs new file mode 100644 index 0000000000..f63e0879b5 --- /dev/null +++ b/backend/windmill-common/tests/dbt_producer_rules.rs @@ -0,0 +1,196 @@ +/*! + * One rule, two spellings: "is dbt the sole producer of this warehouse + * relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription + * is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy + * retroactively leaves unwakeable. Both must answer "yes, dormant" only when + * every script writing the relation is a dbt one — a dbt run does not dispatch — + * and "no" both when a native `// materialize manual dbt://…` producer exists and + * when nothing produces the relation yet, which is the ordinary deploy-order + * case. Every way of getting this wrong is silent: a dormant edge on the canvas, + * a refused deploy of a valid pipeline, or a warning that stops appearing. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer}; + +const WS: &str = "test-workspace"; +const RELATION: &str = "main/analytics/orders"; +const SUBSCRIBER: &str = "u/test-user/consumer"; + +async fn plant_producer(db: &Pool, path: &str, language: &str, hash: i64) { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(language) + .execute(db) + .await + .expect("insert script"); + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING", + ) + .bind(WS) + .bind(RELATION) + .bind(path) + .execute(db) + .await + .expect("insert asset"); +} + +async fn plant_subscriber(db: &Pool, path: &str) { + sqlx::query( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, + trigger_ref) + VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)", + ) + .bind(WS) + .bind(path) + .bind(RELATION) + .execute(db) + .await + .expect("insert script_trigger"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn no_producer_is_not_dormant(db: Pool) { + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None, + "a relation nothing produces yet must not refuse the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn dbt_only_producer_is_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None + ); +} + +/// `asset` is keyed by path while `script` holds every version of it, so the +/// language has to be read off the live one: a path converted to dbt still has +/// its old native versions sitting in `script`. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_superseded_native_version_does_not_count(db: Pool) { + plant_producer(&db, "u/test-user/project", "postgresql", 1).await; + sqlx::query("UPDATE script SET archived = true WHERE hash = 1") + .execute(&db) + .await + .expect("archive the old version"); + plant_producer(&db, "u/test-user/project", "dbt", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +/// The rows of the script being deployed describe the version it replaces, so a +/// script dropping its `// materialize` while adding a subscription would +/// otherwise count itself as the producer that wakes it — and commit a dormant +/// edge. It can never be that producer anyway: the dispatcher skips self-loops. +/// Under a rename that write sits at the OLD path, which the deploy is removing +/// in the same uncommitted transaction, so both paths have to be excluded. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_subscriber_is_never_its_own_producer(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_renamed_producer_is_excluded_too(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer( + &db, + WS, + RELATION, + &[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()] + ) + .await + .unwrap(), + Some("u/test-user/project".to_string()), + "the write this deploy is moving off the old path cannot wake the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_agrees_with_the_singular_one(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, "u/test-user/consumer").await; + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → u/test-user/consumer")], + "dbt alone builds it, so the subscription can never be woken" + ); + + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "a native producer wakes it, so the edge is live" + ); +} + +/// The two ways the set form could stop meaning what the singular one means: a +/// relation nothing produces is deploy order rather than a dormant edge, and a +/// subscriber's own write is not a producer that can wake it — the dispatcher +/// skips self-loops, so that edge is dormant and has to be named. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_matches_on_the_edge_cases_too(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, SUBSCRIBER).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "nothing produces it yet, so nothing is dormant" + ); + + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → {SUBSCRIBER}")], + "the subscriber's own write cannot wake it, so dbt is still the sole producer" + ); +} diff --git a/backend/windmill-common/tests/notify_events.rs b/backend/windmill-common/tests/notify_events.rs index 8161130875..285086308e 100644 --- a/backend/windmill-common/tests/notify_events.rs +++ b/backend/windmill-common/tests/notify_events.rs @@ -360,6 +360,77 @@ async fn test_trigger_notify_workspace_key_change(db: Pool) { ); } +/// The address a job runs as is served from a process-local cache, so every change that can move +/// a `(workspace, username)` -> email mapping has to reach the other replicas as an eviction. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_trigger_notify_user_email_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + sqlx::query("UPDATE usr SET email = 'renamed@windmill.dev' WHERE workspace_id = 'test-workspace' AND username = 'test-user'") + .execute(&db) + .await + .expect("Failed to change email"); + + let events = poll_notify_events(&db, before_id) + .await + .expect("Should poll events"); + assert!( + events.iter().any(|e| e.channel == "notify_user_email_change" + && e.payload == "test-workspace:test-user"), + "email change should evict the key it moved" + ); + + // A superadmin outside their workspaces resolves through `password`, which names no + // workspace: the wildcard is the only way to reach that key. `super_admin` is half of what + // that fallback matches on, so losing it moves the mapping just as the address does. + for (label, stmt, expected_aliases) in [ + ( + "email change", + "UPDATE password SET email = 'sa2@windmill.dev' WHERE email = 'test@windmill.dev'", + // old address, new address, and the username that outlives both + vec!["test@windmill.dev", "sa2@windmill.dev", "test-user"], + ), + ( + "demotion", + "UPDATE password SET super_admin = false WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ( + "promotion", + "UPDATE password SET super_admin = true WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ( + "deletion", + "DELETE FROM password WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ] { + let before_id = get_latest_event_id(&db).await.unwrap(); + sqlx::query(stmt) + .execute(&db) + .await + .unwrap_or_else(|e| panic!("Failed to apply superadmin {label}: {e}")); + + let events = poll_notify_events(&db, before_id) + .await + .expect("Should poll events"); + // Every alias the principal can be spelled as, since `resolve_username_to_email` + // matches a `u/` principal against the username or the address. + let evicted: Vec<&str> = events + .iter() + .filter(|e| e.channel == "notify_user_email_change") + .filter_map(|e| e.payload.strip_prefix("*:")) + .collect(); + for alias in expected_aliases { + assert!( + evicted.contains(&alias), + "superadmin {label} should evict {alias}, got {evicted:?}" + ); + } + } +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_trigger_notify_token_invalidation(db: Pool) { // First insert a session token with token_hash and token_prefix diff --git a/backend/windmill-common/tests/permissioned_as_authz.rs b/backend/windmill-common/tests/permissioned_as_authz.rs new file mode 100644 index 0000000000..25c219845f --- /dev/null +++ b/backend/windmill-common/tests/permissioned_as_authz.rs @@ -0,0 +1,81 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::auth::fetch_authed_from_permissioned_as; + +/// The address handed to `fetch_authed_from_permissioned_as` may come from a cache that a +/// username reassignment has outrun. It must not be believed: the workspace role is keyed on the +/// principal while `super_admin` and `email_to_igroup` are keyed on the address, so trusting a +/// stale one would run the new holder's job with the previous holder's instance privileges. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_stale_address_cannot_carry_the_previous_holders_privileges(db: Pool) { + // `test-user` in the fixture is a superadmin with the address `test@windmill.dev`. Free the + // username and hand it to somebody who is not, exactly as an offboard-then-onboard would. + sqlx::query("DELETE FROM usr WHERE workspace_id = 'test-workspace' AND username = 'test-user'") + .execute(&db) + .await + .expect("free the username"); + sqlx::query( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('newcomer@windmill.dev', 'x', 'password', false, true, 'Newcomer')", + ) + .execute(&db) + .await + .expect("create the new account"); + sqlx::query( + "INSERT INTO usr(workspace_id, email, username, is_admin, role) + VALUES ('test-workspace', 'newcomer@windmill.dev', 'test-user', false, 'User')", + ) + .execute(&db) + .await + .expect("reassign the username"); + + // What a replica that has not yet consumed the eviction would pass: the principal is the + // reassigned username, the address is the one it cached for the previous holder. + let authed = fetch_authed_from_permissioned_as( + "u/test-user", + "test@windmill.dev", + "test-workspace", + &db, + ) + .await + .expect("should authenticate the current holder"); + + assert_eq!( + authed.email, "newcomer@windmill.dev", + "the principal's live address must win over the one supplied" + ); + assert!( + !authed.is_admin, + "the new holder must not inherit the previous holder's superadmin" + ); +} + +/// A disabled member still holds its username in the workspace. Workspace usernames are only +/// unique per workspace, so an unrelated instance superadmin can share it, and falling through to +/// the `password` fallback would run the disabled member's jobs as that superadmin. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_disabled_member_never_resolves_to_a_same_named_superadmin(db: Pool) { + sqlx::query( + "UPDATE usr SET disabled = true WHERE workspace_id = 'test-workspace' AND username = 'test-user-2'", + ) + .execute(&db) + .await + .expect("disable the member"); + sqlx::query( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('other-superadmin@windmill.dev', 'x', 'password', true, true, 'Other', 'test-user-2')", + ) + .execute(&db) + .await + .expect("create the same-named superadmin"); + + for supplied in ["test2@windmill.dev", "other-superadmin@windmill.dev"] { + let authed = + fetch_authed_from_permissioned_as("u/test-user-2", supplied, "test-workspace", &db) + .await; + assert!( + authed.is_err(), + "a disabled member must not authenticate (supplied {supplied}): {:?}", + authed.map(|a| (a.email, a.is_admin)) + ); + } +} diff --git a/backend/windmill-common/tests/queue_metrics_series.rs b/backend/windmill-common/tests/queue_metrics_series.rs new file mode 100644 index 0000000000..f6bc7b4056 --- /dev/null +++ b/backend/windmill-common/tests/queue_metrics_series.rs @@ -0,0 +1,136 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::queue_metrics::{read_queue_metrics_series, QUEUE_METRIC_STALE_SECS}; + +const WINDOW: f64 = 3600.0; + +/// Store a sample written `at` seconds after the start of a `WINDOW` ending now. +async fn sample(db: &Pool, id: &str, value: serde_json::Value, at: f64) { + sqlx::query( + "INSERT INTO metrics (id, value, created_at) VALUES ($1, $2, now() - make_interval(secs => $3))", + ) + .bind(id) + .bind(value) + .bind(WINDOW - at) + .execute(db) + .await + .expect("failed to store a metric sample"); +} + +/// The database hands the renderer the last sample before the window, which sets the left edge, +/// and for each slot its peak and its latest value, which the line continues from. +#[sqlx::test(migrations = "../migrations")] +async fn a_series_starts_from_the_sample_before_the_window_and_keeps_each_slot_peak( + db: Pool, +) { + // Before the window: 1, then 2, which is what is in force at the left edge. + sample(&db, "queue_count_t", json!(1), -120.0).await; + sample(&db, "queue_count_t", json!(2), -60.0).await; + // Three samples inside one 30s slot: the line rises to their peak, then drops to the last. + sample(&db, "queue_count_t", json!(5), 605.0).await; + sample(&db, "queue_count_t", json!(9), 612.0).await; + sample(&db, "queue_count_t", json!(4), 620.0).await; + // Drained before the window: nothing left to draw. + sample(&db, "queue_count_gone", json!(3), -300.0).await; + sample(&db, "queue_count_gone", json!(0), -200.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + + assert_eq!( + series.tags.len(), + 1, + "a tag drained before the window is left out" + ); + let tag = &series.tags[0]; + assert_eq!(tag.tag, "t"); + assert!(tag.delay.is_empty()); + + let stale = 620.0 + QUEUE_METRIC_STALE_SECS; + let expected = [ + (0.0, 2.0), + (605.0, 2.0), + (605.0, 9.0), + (620.0, 9.0), + (620.0, 4.0), + (stale, 4.0), + (stale, 0.0), + (WINDOW, 0.0), + ]; + assert_eq!(tag.count.len(), expected.len(), "vertices: {:?}", tag.count); + for ((ms, value), (at, expected_value)) in tag.count.iter().zip(expected) { + let secs = (*ms - series.from) as f64 / 1000.0; + // Samples are stored a few milliseconds before the window is read. + assert!( + (secs - at).abs() < 2.0 && *value == expected_value, + "expected ({at}, {expected_value}), got ({secs}, {value}) in {:?}", + tag.count + ); + } +} + +/// A delay stored as its head's wait start is drawn as that wait, growing a second per second, +/// right up to the zero that closes it. +#[sqlx::test(migrations = "../migrations")] +async fn a_climbing_delay_is_drawn_as_the_wait_of_its_head(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // The head started waiting 30s before the window; heartbeats restate it until the drain. + let head = json!({ "since": now - WINDOW - 30.0 }); + for at in [60.0, 360.0, 660.0] { + sample(&db, "queue_delay_t", head.clone(), at).await; + } + sample(&db, "queue_delay_t", json!(0), 900.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let points = series.tags[0] + .delay + .iter() + .map(|(ms, value)| ((*ms - series.from) as f64 / 1000.0, *value)) + .collect::>(); + + let climb = points + .iter() + .filter(|(_, value)| *value > 0.0) + .collect::>(); + assert!( + climb.len() > 4, + "the climb has vertices along the way: {points:?}" + ); + for (at, value) in &climb { + assert!( + (value - (at + 30.0)).abs() < 2.0, + "off the climb at {at}: {points:?}" + ); + } + let (first, _) = climb[0]; + let (top, _) = climb[climb.len() - 1]; + assert!( + (first - 60.0).abs() < 2.0 && (top - 900.0).abs() < 2.0, + "{points:?}" + ); +} + +/// A climb that drains inside its slot keeps its top, which no stored value holds: it is reached +/// at the next sample. +#[sqlx::test(migrations = "../migrations")] +async fn a_climb_that_drains_inside_its_slot_keeps_its_top(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // All in the 30s slot starting at 600: held at 5s, then climbing from a head queued at 597, + // which is still there when the tag drains at 627, 30s into its wait. + sample(&db, "queue_delay_t", json!(5), 602.0).await; + sample(&db, "queue_delay_t", json!({ "since": now - WINDOW + 597.0 }), 610.0).await; + sample(&db, "queue_delay_t", json!(0), 627.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let top = series.tags[0] + .delay + .iter() + .map(|(_, value)| *value) + .fold(0.0, f64::max); + assert!((top - 30.0).abs() < 2.0, "{:?}", series.tags[0].delay); +} diff --git a/backend/windmill-common/tests/queue_stats.rs b/backend/windmill-common/tests/queue_stats.rs new file mode 100644 index 0000000000..419ad98d56 --- /dev/null +++ b/backend/windmill-common/tests/queue_stats.rs @@ -0,0 +1,69 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::queue::get_queue_stats; + +const WORKSPACE: &str = "test-workspace"; + +async fn queue_job( + db: &Pool, + tag: &str, + priority: Option, + waited_secs: f64, + running: bool, +) { + sqlx::query( + "WITH job AS ( + INSERT INTO v2_job (id, workspace_id, tag) VALUES (gen_random_uuid(), $1, $2) + RETURNING id + ) + INSERT INTO v2_job_queue (id, workspace_id, tag, priority, running, scheduled_for) + SELECT id, $1, $2, $3, $4, now() - make_interval(secs => $5) FROM job", + ) + .bind(WORKSPACE) + .bind(tag) + .bind(priority) + .bind(running) + .bind(waited_secs) + .execute(db) + .await + .expect("failed to queue job"); +} + +/// The delay reported for a tag is that of the job the worker pull takes first, ordered +/// `priority DESC NULLS LAST, scheduled_for`, not simply the oldest one waiting. Running jobs +/// and jobs less than 3 seconds past due are not part of the backlog at all. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn queue_stats_report_the_delay_of_the_job_pulled_next(db: Pool) { + // The oldest job has no priority, so every prioritized job runs before it. + queue_job(&db, "mixed", None, 900.0, false).await; + queue_job(&db, "mixed", Some(1), 600.0, false).await; + queue_job(&db, "mixed", Some(5), 300.0, false).await; + queue_job(&db, "mixed", Some(5), 100.0, false).await; + // Highest priority, but not backlog: already running, or not yet 3 seconds past due. + queue_job(&db, "mixed", Some(9), 1200.0, true).await; + queue_job(&db, "mixed", Some(9), 1.0, false).await; + queue_job(&db, "unprioritized", None, 500.0, false).await; + queue_job(&db, "unprioritized", None, 50.0, false).await; + + let stats = get_queue_stats(&db).await.unwrap(); + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + + let mixed = &stats["mixed"]; + assert_eq!(mixed.count, 4); + assert!( + (mixed.delay - 300.0).abs() < 5.0, + "expected the oldest job of the highest priority, got a delay of {}", + mixed.delay + ); + // The same job's wait start, which the delay is measured from. + assert!((mixed.head_since + mixed.delay - now).abs() < 5.0); + let unprioritized = &stats["unprioritized"]; + assert_eq!(unprioritized.count, 2); + assert!( + (unprioritized.delay - 500.0).abs() < 5.0, + "expected the oldest job, got a delay of {}", + unprioritized.delay + ); +} diff --git a/backend/windmill-common/tests/schema_replica_identity.rs b/backend/windmill-common/tests/schema_replica_identity.rs new file mode 100644 index 0000000000..fb14a5c78e --- /dev/null +++ b/backend/windmill-common/tests/schema_replica_identity.rs @@ -0,0 +1,46 @@ +//! Every table must be replicable. +//! +//! PostgreSQL refuses UPDATE and DELETE on a table that has neither a PRIMARY KEY +//! nor an explicit REPLICA IDENTITY once the database is published to a logical +//! replication slot. That is what a low-downtime major-version upgrade runs on +//! (RDS and Aurora Blue/Green, pglogical) and what every CDC pipeline reads, so a +//! single keyless table blocks the upgrade outright. This runs against a freshly +//! migrated database and fails on the migration that introduces one. + +use sqlx::{Pool, Postgres}; + +/// Partitioned parents are checked alongside ordinary tables: a parent without a +/// key hands the same defect to every partition created under it later. +#[sqlx::test(migrations = "../migrations")] +async fn every_table_is_replicable(db: Pool) -> anyhow::Result<()> { + let offenders: Vec = sqlx::query_scalar( + "SELECT n.nspname || '.' || c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND NOT ( + -- FULL and USING INDEX replicate on their own. + c.relreplident IN ('f', 'i') + -- DEFAULT resolves to the primary key, so it needs one to exist. + -- NOTHING never replicates, primary key or not. + OR (c.relreplident = 'd' AND EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary + )) + ) + ORDER BY 1", + ) + .fetch_all(&db) + .await?; + + assert!( + offenders.is_empty(), + "logical replication will reject UPDATE and DELETE on these tables, because \ + they carry no replica identity it can use: {}. \ + Give each one a primary key -- a natural composite key where every column \ + is NOT NULL, otherwise a surrogate `BIGINT GENERATED ALWAYS AS IDENTITY`.", + offenders.join(", ") + ); + + Ok(()) +} diff --git a/backend/windmill-dep-map/src/ci_tests.rs b/backend/windmill-dep-map/src/ci_tests.rs index 79f1b7e30b..8f56111b35 100644 --- a/backend/windmill-dep-map/src/ci_tests.rs +++ b/backend/windmill-dep-map/src/ci_tests.rs @@ -19,3 +19,13 @@ pub async fn trigger_ci_tests_for_item( ) -> error::Result> { Ok(vec![]) } + +#[cfg(not(feature = "private"))] +pub async fn trigger_all_ci_tests( + _db: &sqlx::Pool, + _w_id: &str, + _email: &str, + _username: &str, +) -> error::Result> { + Ok(vec![]) +} diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs index 650e0e5f99..aab50795a9 100644 --- a/backend/windmill-dep-map/src/lib.rs +++ b/backend/windmill-dep-map/src/lib.rs @@ -128,6 +128,40 @@ pub fn extract_referenced_paths( } } +/// Re-records which paths `script_path` imports and what each one's lock hashes to right now. +/// That snapshot is what a later relock-skip check of this importer compares against, so it +/// has to move whenever the imports may have, whether or not the importer's own lock did. +/// +/// Writes for any path in `w_id` and checks nothing: callers are responsible for having +/// established access to that workspace and script, as a dependency job's push already has. +pub async fn refresh_dependency_map( + db: &sqlx::Pool, + w_id: &str, + script_path: &str, + parent_path: &Option, + code: &str, + script_lang: &Option, +) -> error::Result<()> { + use scoped_dependency_map::ScopedDependencyMap; + + let mut tx = db.begin().await?; + let mut dependency_map = + ScopedDependencyMap::fetch_maybe_rearranged(w_id, script_path, "script", parent_path, db) + .await?; + + tx = dependency_map + .patch( + extract_referenced_paths(code, script_path, *script_lang), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, + ) + .await?; + + dependency_map.dissolve(tx).await.commit().await?; + Ok(()) +} + pub async fn process_relative_imports( db: &sqlx::Pool, _job_id: Option, @@ -145,29 +179,7 @@ pub async fn process_relative_imports( use scoped_dependency_map::ScopedDependencyMap; use trigger_dependents::trigger_dependents_to_recompute_dependencies; - // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled - { - let mut tx = db.begin().await?; - let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( - &w_id, - script_path, - "script", - &parent_path, - db, - ) - .await?; - - tx = dependency_map - .patch( - extract_referenced_paths(&code, script_path, *script_lang), - // Ideally should be None, but due to current implementation will use empty string to represent None. - "".into(), - tx, - ) - .await?; - - dependency_map.dissolve(tx).await.commit().await?; - } + refresh_dependency_map(db, w_id, script_path, &parent_path, code, script_lang).await?; { let mut already_visited = args diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index 148746dcca..f74581ba67 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,7 +9,7 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] -private = ["windmill-common/private"] +private = ["windmill-common/private", "windmill-dep-map/private"] enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] all_sqlx_features = ["enterprise"] default = [] @@ -22,5 +22,6 @@ serde_json.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true +windmill-dep-map.workspace = true regex = "1.10.3" tokio = { workspace = true, features = ["full"] } \ No newline at end of file diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 2bfc7d2f77..d65e230988 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -14,12 +14,21 @@ pub mod git_sync_oss; #[cfg(feature = "private")] pub use git_sync_ee::{ - enqueue_git_pull_dry_run, enqueue_git_pull_job, handle_deployment_metadata, - handle_deployment_metadata_batch, handle_fork_branch_creation, persist_auto_pull_state, - reconcile_and_enqueue_pull, reconcile_fork_branch_pull, record_auto_pull_failure, + clear_auto_pull_failure, enqueue_git_pull_dry_run, enqueue_git_pull_job, + handle_deployment_metadata, handle_deployment_metadata_batch, handle_fork_branch_creation, + persist_auto_pull_state, reconcile_and_enqueue_pull, reconcile_fork_branch_pull, + record_auto_pull_failure, record_synced_head, sweep_ci_test_checks, tally_deployed_object_changes, }; +// The CI-test check exists only on enterprise builds; `private` alone (the CE image) +// compiles git_sync_ee without them. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub use git_sync_ee::{ + ensure_ci_test_check_for_pr, evaluate_and_conclude_ci_test_checks, + post_ci_test_check_not_applicable, resolve_pr_head_workspace, +}; + #[cfg(not(feature = "private"))] pub use git_sync_oss::{ handle_deployment_metadata, handle_deployment_metadata_batch, handle_fork_branch_creation, diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index cc6c61be81..27a50ba07a 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -1,8 +1,8 @@ use crate::{ classify_read_failure, decrypt_oauth_data, delete_native_trigger, delete_token_by_hash, get_native_trigger, list_native_triggers, lock::TriggerLock, map_external_error, - map_external_error_with, rotate_webhook_token, store_native_trigger, - sync::EXTERNAL_TRIGGER_MISSING_ERROR, update_native_trigger_error, + map_external_error_with, rotate_webhook_token, set_native_trigger_enabled, + store_native_trigger, sync::EXTERNAL_TRIGGER_MISSING_ERROR, update_native_trigger_error, update_native_trigger_if_runnable_unchanged, webhook_token_label, webhook_token_scopes, External, ExternalReadFailure, NativeTrigger, NativeTriggerConfig, NativeTriggerData, ServiceName, @@ -239,6 +239,7 @@ async fn create_native_trigger( &config, service_config, data.summary.as_deref(), + data.enabled, ) .await?; @@ -603,6 +604,87 @@ async fn delete_native_trigger_handler( Ok(format!("Native trigger deleted")) } +#[derive(Debug, Deserialize)] +pub struct SetEnabledPayload { + pub enabled: bool, +} + +/// Pause or resume a trigger, without touching its registration on the external service. +/// +/// Leaving the webhook registered is what makes this reversible: services drop or deactivate a +/// subscription that keeps failing, so a paused trigger keeps answering deliveries normally and +/// simply starts no job. +async fn set_native_trigger_enabled_handler( + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, external_id)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) + .await? + .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; + + check_scopes(&authed, || { + format!("native_triggers:write:{}", &existing.script_path) + })?; + require_is_writer_on_runnable( + &authed, + &existing.script_path, + existing.is_flow, + &workspace_id, + db.clone(), + ) + .await?; + + let updated = set_native_trigger_enabled( + &mut *tx, + &workspace_id, + service_name, + &external_id, + payload.enabled, + ) + .await?; + + // The read above takes no row lock, so a concurrent delete can land in between; reporting + // success then would tell the caller a trigger that is gone had been paused. + if !updated { + return Err(Error::NotFound(format!( + "Native trigger not found: {}", + external_id + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!( + "native_triggers.{}.{}", + service_name, + if payload.enabled { "enable" } else { "disable" } + ), + ActionKind::Update, + &workspace_id, + Some(&external_id), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!( + "Native trigger {}", + if payload.enabled { + "enabled" + } else { + "disabled" + } + )) +} + async fn list_native_triggers_handler( Extension(service_name): Extension, authed: ApiAuthed, @@ -642,6 +724,10 @@ pub fn service_routes(handler: T) -> Router { .route( "/delete/{external_id}", delete(delete_native_trigger_handler::), + ) + .route( + "/setenabled/{external_id}", + post(set_native_trigger_enabled_handler::), ); standard_routes diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 4400091ed6..570319ee0e 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -226,6 +226,10 @@ pub struct NativeTrigger { pub created_at: DateTime, pub updated_at: DateTime, pub summary: Option, + /// Whether incoming webhooks for this trigger start a job. Operational state: a create sets + /// its initial value and `setenabled` is its only mutator afterwards, so saving a + /// configuration can never silently re-enable a trigger someone paused. + pub enabled: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -235,12 +239,20 @@ pub struct NativeTriggerConfig { pub webhook_token: String, } +fn default_true() -> bool { + true +} + #[derive(Debug, Serialize, Deserialize)] pub struct NativeTriggerData { pub script_path: String, pub is_flow: bool, pub service_config: C, pub summary: Option, + /// Honoured on create only, so a trigger can be registered already paused in one request. + /// An update ignores it: `setenabled` is the only way to change an existing trigger's state. + #[serde(default = "default_true")] + pub enabled: bool, } #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] @@ -1179,11 +1191,15 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> config: &NativeTriggerConfig, service_config: C, summary: Option<&str>, + enabled: bool, ) -> Result<()> { use windmill_common::auth::hash_token; let webhook_token_hash = hash_token(&config.webhook_token); + // `enabled` is set by the INSERT alone: writing it here rather than in a follow-up statement + // is what keeps a trigger created paused from ever being visible, and therefore runnable, in + // any other state. The conflict branch leaves it untouched for the mirror-image reason. sqlx::query!( r#" INSERT INTO native_trigger ( @@ -1194,9 +1210,10 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> is_flow, webhook_token_hash, service_config, - summary + summary, + enabled ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8 + $1, $2, $3, $4, $5, $6, $7, $8, $9 ) ON CONFLICT (external_id, workspace_id, service_name) DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW() @@ -1209,6 +1226,7 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> webhook_token_hash, sqlx::types::Json(service_config) as _, summary, + enabled, ) .execute(db) .await?; @@ -1405,7 +1423,8 @@ pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( error, created_at, updated_at, - summary + summary, + enabled FROM native_trigger WHERE @@ -1444,7 +1463,8 @@ pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = P error, created_at, updated_at, - summary + summary, + enabled FROM native_trigger WHERE @@ -1491,7 +1511,8 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> nt.error, nt.created_at, nt.updated_at, - nt.summary + nt.summary, + nt.enabled FROM native_trigger nt WHERE @@ -1555,6 +1576,71 @@ pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Po Ok(()) } +/// Pause or resume a trigger. Returns `false` when there is no such trigger. +/// +/// Callers MUST have verified write access to the trigger's runnable: this writes operational +/// state and performs no authorization of its own. +pub async fn set_native_trigger_enabled<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + enabled: bool, +) -> Result { + // `updated_at` is the row version `record_reregistration` conditions on, so leave it alone: + // pausing a trigger must not make a registration that is mid-flight discard its result. + let updated = sqlx::query!( + r#" + UPDATE native_trigger + SET enabled = $1 + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + enabled, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await? + .rows_affected(); + + Ok(updated > 0) +} + +/// Whether a webhook arriving for this trigger should start a job. +/// +/// A trigger Windmill no longer knows about counts as enabled: the token in the URL is what +/// authorizes the run, and this is a pause switch, not a second authorization check. It reads +/// nothing a caller could not already learn from the trigger it is delivering for, so it needs no +/// authorization of its own — but it also grants none, and must not be used as one. +pub async fn native_trigger_is_enabled<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result { + let enabled = sqlx::query_scalar!( + r#" + SELECT enabled + FROM native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id, + ) + .fetch_optional(db) + .await?; + + Ok(enabled.unwrap_or(true)) +} + pub async fn update_native_trigger_service_config< 'c, E: sqlx::Executor<'c, Database = Postgres>, @@ -1701,6 +1787,11 @@ pub async fn delete_workspace_integration( /// /// `external_id` is optional because during CREATE we don't have it yet /// (it's returned by the external service). During UPDATE, we have it. +/// +/// Every registered URL MUST end up carrying it: it is the only thing a delivery identifies its +/// trigger by, so a service that leaves it out ships a disable switch that silently does nothing. +/// A handler that returns `None` from `service_config_from_create_response` gets this for free — +/// `create_native_trigger` then runs the `update` cycle that re-registers with the assigned id. pub fn generate_webhook_service_url( base_url: &str, w_id: &str, diff --git a/backend/windmill-native-triggers/src/rename.rs b/backend/windmill-native-triggers/src/rename.rs index a7168f5c7e..caae99028c 100644 --- a/backend/windmill-native-triggers/src/rename.rs +++ b/backend/windmill-native-triggers/src/rename.rs @@ -135,6 +135,7 @@ async fn reregister_one( is_flow: trigger.is_flow, service_config, summary: trigger.summary.clone(), + enabled: trigger.enabled, }; // The token is scoped to the runnable path and only its hash is kept, so pointing the webhook diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index 2650b68b4d..a8a8e83883 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -88,6 +88,10 @@ pub struct OAuthConfig { #[serde(default = "empty_string")] pub token_url: String, pub userinfo_url: Option, + /// The registry JSON may also carry two frontend-only keys for the connect + /// dialog, deliberately not modelled here: `scope_options`, a scope pick + /// list, and `resource_fields`, the fields of the resource type the dialog + /// asks for once the token is in (Snowflake's database and warehouse). pub scopes: Option>, /// Default scopes for the client-credentials (2-legged) flow. These differ /// from the authorization-code `scopes` for most providers (member/consent diff --git a/backend/windmill-object-store/Cargo.toml b/backend/windmill-object-store/Cargo.toml index 3602007d8c..90af4f7e7c 100644 --- a/backend/windmill-object-store/Cargo.toml +++ b/backend/windmill-object-store/Cargo.toml @@ -58,3 +58,4 @@ aws-credential-types = { workspace = true, optional = true } tempfile.workspace = true tokio = { workspace = true, features = ["rt", "macros"] } object_store.workspace = true +serial_test = "3" diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index cc588d485f..c403625e00 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -162,37 +162,211 @@ impl From> for ExpirableObjectStore { #[cfg(feature = "parquet")] lazy_static::lazy_static! { pub static ref OBJECT_STORE_SETTINGS: Arc>> = Arc::new(RwLock::new(None)); + + /// Worker-group override of the store backing the *dependency cache* only: venvs, language + /// bundles and compiled binaries, which a worker both writes and reads back itself. + /// Everything the server also reads — job results, logs, codebases, app assets — stays on + /// [`OBJECT_STORE_SETTINGS`], which a worker-local redirect would make unreachable. + static ref CACHE_OBJECT_STORE_OVERRIDE: Arc>> = Arc::new(RwLock::new(None)); + + /// The config [`CACHE_OBJECT_STORE_OVERRIDE`] was built from, so a rebuild that fails for a + /// config already being served can keep serving it. Locked after the store, never before. + static ref CACHE_OVERRIDE_APPLIED: Arc>> = Arc::new(RwLock::new(None)); + + /// Held across a whole [`reload_cache_object_store_override`], build included, so that the + /// override's flag, store and applied config only ever move together. + static ref CACHE_OVERRIDE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(()); +} + +/// Whether a worker-group cache override is configured, held apart from the store it built so +/// that a configured-but-broken override reads as "no cache store" instead of silently falling +/// back to the instance bucket the operator redirected away from. +#[cfg(feature = "parquet")] +static CACHE_OBJECT_STORE_OVERRIDDEN: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Bumped by every [`reload_cache_object_store_override`] at entry. Builds are slow and several +/// callers race — a config change, the retry behind a failed one, a license-plan change — and the +/// lock alone would only order them by arrival, so a reload that lost its claim while waiting +/// drops out rather than installing a store the group has already moved off. +#[cfg(feature = "parquet")] +static CACHE_OVERRIDE_GENERATION: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +#[cfg(feature = "parquet")] +async fn resolve_object_store( + settings_lock: &RwLock>, +) -> Option> { + let settings = settings_lock.read().await; + let Some(s) = settings.as_ref() else { + return None; + }; + match &s.refresh { + Some(refresh) if refresh.refresh_needed() => { + let refresh = refresh.clone(); + let refreshed_from = s.store.clone(); + drop(settings); + let new_store = refresh.refresh().await?; + let mut settings = settings_lock.write().await; + match settings.as_ref() { + // A reload may have installed a different store while the credentials were + // being minted; that one reflects newer config, so the refresh is stale. + Some(current) if !Arc::ptr_eq(¤t.store, &refreshed_from) => { + Some(current.store.clone()) + } + Some(_) => { + let arc = new_store.store.clone(); + *settings = Some(new_store); + Some(arc) + } + // Cleared while refreshing. + None => None, + } + } + _ => Some(s.store.clone()), + } } #[cfg(feature = "parquet")] pub async fn get_object_store() -> Option> { - let settings = OBJECT_STORE_SETTINGS.read().await; - if let Some(s) = settings.as_ref() { - match &s.refresh { - Some(refresh) => { - if refresh.refresh_needed() { - let refresh = refresh.clone(); - drop(settings); - let new_store = refresh.refresh().await; - if let Some(new_store) = new_store { - let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await; - let arc = new_store.store.clone(); - *s3_cache_settings = Some(new_store); - return Some(arc); - } else { - return None; - } - } else { - return Some(s.store.clone()); - } - } - None => { - return Some(s.store.clone()); - } - } - } else { - return None; + resolve_object_store(&OBJECT_STORE_SETTINGS).await +} + +/// The store the dependency cache reads and writes: the worker group's override when it has one, +/// the instance object store otherwise. Anything the server must also reach goes through +/// [`get_object_store`] instead. +#[cfg(feature = "parquet")] +pub async fn get_cache_object_store() -> Option> { + if CACHE_OBJECT_STORE_OVERRIDDEN.load(std::sync::atomic::Ordering::Relaxed) { + return resolve_object_store(&CACHE_OBJECT_STORE_OVERRIDE).await; } + resolve_object_store(&OBJECT_STORE_SETTINGS).await +} + +/// True when an override is configured but has no usable store. The caller's short retry is the +/// fast path back; this is the backstop, and the full settings reload it rides on is 12h apart by +/// default (`SETTINGS_RELOAD_PERIOD_SECS`), so an outage outlasting the retry keeps the group's +/// dependency cache local until then or until someone edits the group config. +#[cfg(feature = "parquet")] +pub async fn cache_object_store_override_failed() -> bool { + CACHE_OBJECT_STORE_OVERRIDDEN.load(std::sync::atomic::Ordering::Relaxed) + && CACHE_OBJECT_STORE_OVERRIDE.read().await.is_none() +} + +/// Apply the `object_store_cache_config` of this worker's group. `None` (or JSON null) drops the +/// override and returns the worker to the instance object store. +/// +/// Returns [`ObjectStoreReload::Later`] when the store did not build for a reason that may pass — +/// the caller is expected to retry shortly, as `initial_load` does for the instance store. +#[cfg(feature = "parquet")] +pub async fn reload_cache_object_store_override( + db: &windmill_common::DB, + settings: Option, +) -> ObjectStoreReload { + use std::sync::atomic::Ordering; + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + // Claim a generation, then take the lock: every state transition below happens inside one + // critical section, and a caller that lost its claim while waiting drops out rather than + // installing what the group has already moved off. + let generation = CACHE_OVERRIDE_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; + let _transition = CACHE_OVERRIDE_LOCK.lock().await; + if CACHE_OVERRIDE_GENERATION.load(Ordering::SeqCst) != generation { + return ObjectStoreReload::Never; + } + + // DISABLE_S3_STORE turns off the instance object store for this process; a group override + // must not be a way back in. + let store_disabled = std::env::var("DISABLE_S3_STORE") + .ok() + .is_some_and(|x| x == "1" || x == "true"); + + let Some(settings) = settings.filter(|v| !v.is_null() && !store_disabled) else { + if CACHE_OBJECT_STORE_OVERRIDDEN.swap(false, Ordering::Relaxed) { + clear_cache_object_store_override().await; + tracing::info!( + "Worker group object store cache override removed, falling back to the instance object store" + ); + } + return ObjectStoreReload::Never; + }; + + // Enterprise-only, so anything else — Community, including a CE build reaching this through + // the config-as-code API, and Pro — must not get a store, and a plan that stops being + // Enterprise must drop one loaded while it still was. + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + tracing::error!( + "Object store cache override requires an enterprise license, ignoring it for this worker group" + ); + if CACHE_OBJECT_STORE_OVERRIDDEN.swap(false, Ordering::Relaxed) { + clear_cache_object_store_override().await; + } + return ObjectStoreReload::Never; + } + + apply_cache_object_store_override(db, settings).await +} + +/// The half of [`reload_cache_object_store_override`] past the entitlement gate: build the store +/// and commit it. Split out so the commit rules are testable without a license plan. +#[cfg(feature = "parquet")] +async fn apply_cache_object_store_override( + db: &windmill_common::DB, + settings: serde_json::Value, +) -> ObjectStoreReload { + use std::sync::atomic::Ordering; + + // Claim the override before building it: until a store is in place the dependency cache + // must stay local-only rather than reach for the instance bucket. + CACHE_OBJECT_STORE_OVERRIDDEN.store(true, Ordering::Relaxed); + + let (store, reload) = match serde_json::from_value::(settings.clone()) { + Ok(setting) => match build_object_store_from_settings(setting, Some(db)).await { + Ok(store) => (Some(store), ObjectStoreReload::Never), + Err(e) => { + tracing::error!( + "Error building the worker group object store cache override, the dependency cache stays local to this worker until it builds: {e:?}" + ); + (None, ObjectStoreReload::Later) + } + }, + // A malformed config will read the same on every retry. + Err(e) => { + tracing::error!( + "Error parsing the worker group object store cache override, the dependency cache stays local to this worker: {e:?}" + ); + (None, ObjectStoreReload::Never) + } + }; + + let mut current = CACHE_OBJECT_STORE_OVERRIDE.write().await; + match store { + Some(store) => { + *current = Some(store); + *CACHE_OVERRIDE_APPLIED.write().await = Some(settings); + tracing::info!( + "Dependency cache of this worker group now uses its own object store, not the instance one" + ); + } + // A rebuild that failed for the config already being served leaves that store in place: + // the group is entitled to it, and dropping it would take the whole group's cache local + // over a transient error. A *different* config failing must still clear, or the worker + // would keep writing to the bucket the operator redirected it away from. + None if current.is_some() + && CACHE_OVERRIDE_APPLIED.read().await.as_ref() == Some(&settings) => {} + None => { + *current = None; + *CACHE_OVERRIDE_APPLIED.write().await = None; + } + } + reload +} + +/// Drop the override store and the config it was built from, in that lock order. +#[cfg(feature = "parquet")] +async fn clear_cache_object_store_override() { + *CACHE_OBJECT_STORE_OVERRIDE.write().await = None; + *CACHE_OVERRIDE_APPLIED.write().await = None; } #[cfg(feature = "parquet")] @@ -2361,10 +2535,100 @@ mod tests { .contains("Error building filesystem object store")); } + /// A worker group override that is configured but has no usable store must leave the + /// dependency cache with no object store at all. Falling back to the instance one would + /// write the group's cache into the bucket the operator redirected it away from. + // Serialized with the other test that swaps OBJECT_STORE_SETTINGS: the store is + // process-global and CI runs this binary with --test-threads=10. + #[cfg(feature = "parquet")] + #[tokio::test] + #[serial_test::serial(object_store_settings)] + async fn test_get_cache_object_store_override() { + use object_store::{path::Path, ObjectStore, PutPayload}; + use std::sync::atomic::Ordering; + + async fn marker_of(store: &Arc) -> String { + let bytes = store.get(&Path::from("marker")).await.unwrap(); + String::from_utf8(bytes.bytes().await.unwrap().to_vec()).unwrap() + } + + let instance_dir = tempfile::tempdir().unwrap(); + let instance = build_filesystem_client(instance_dir.path().to_str().unwrap()).unwrap(); + instance + .put(&Path::from("marker"), PutPayload::from("instance")) + .await + .unwrap(); + + let group_dir = tempfile::tempdir().unwrap(); + let group = build_filesystem_client(group_dir.path().to_str().unwrap()).unwrap(); + group + .put(&Path::from("marker"), PutPayload::from("group")) + .await + .unwrap(); + + *OBJECT_STORE_SETTINGS.write().await = Some(ExpirableObjectStore::from(instance)); + + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "instance"); + assert!(!cache_object_store_override_failed().await); + + CACHE_OBJECT_STORE_OVERRIDDEN.store(true, Ordering::Relaxed); + *CACHE_OBJECT_STORE_OVERRIDE.write().await = Some(ExpirableObjectStore::from(group)); + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "group"); + + // Configured but unbuilt, as a failed reload leaves it: no store at all, rather than the + // instance bucket the operator redirected the group away from. + *CACHE_OBJECT_STORE_OVERRIDE.write().await = None; + assert!(get_cache_object_store().await.is_none()); + assert!(cache_object_store_override_failed().await); + + // The teardown branch returns before the pool is used, so a lazy one is enough. + let db = sqlx::postgres::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); + reload_cache_object_store_override(&db, None).await; + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "instance"); + assert!(!cache_object_store_override_failed().await); + + *OBJECT_STORE_SETTINGS.write().await = None; + } + + /// A rebuild is triggered by any edit to the group config, not only by editing the store, so + /// a build that fails for the config already installed must leave it alone — otherwise a + /// renamed worker tag plus one flaky token mint takes the whole group's cache local. A + /// *different* config failing still has to clear it. + #[cfg(feature = "parquet")] + #[tokio::test] + #[serial_test::serial(object_store_settings)] + async fn test_failed_rebuild_keeps_the_store_serving_the_same_config() { + let db = sqlx::postgres::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); + let dir = tempfile::tempdir().unwrap(); + let settings = serde_json::json!({ + "type": "Filesystem", "root_path": dir.path().to_str().unwrap() + }); + + apply_cache_object_store_override(&db, settings.clone()).await; + assert!(get_cache_object_store().await.is_some()); + + // Same config, now unbuildable: the store it already produced stays. + dir.close().unwrap(); + apply_cache_object_store_override(&db, settings).await; + assert!(get_cache_object_store().await.is_some()); + + // A different config that will not build must not leave the old bucket in place. + let moved = serde_json::json!({ "type": "Filesystem", "root_path": "/proc/nonexistent" }); + apply_cache_object_store_override(&db, moved).await; + assert!(get_cache_object_store().await.is_none()); + assert!(cache_object_store_override_failed().await); + + reload_cache_object_store_override(&db, None).await; + } + // --- get_logs_from_store test --- #[cfg(feature = "parquet")] #[tokio::test] + #[serial_test::serial(object_store_settings)] async fn test_get_logs_from_store_with_filesystem() { use futures::StreamExt; use object_store::{path::Path, ObjectStore, PutPayload}; diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index b9bd6fd28e..6dc5d8ba42 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" default = [] private = [] enterprise = ["windmill-common/enterprise"] -cloud = [] +cloud = ["windmill-common/cloud"] benchmark = ["windmill-common/benchmark"] failpoints = [] prometheus = ["dep:prometheus"] diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index af9ca957b9..a3f7154cda 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result // A dbt run records the relations it builds, so it looks like a producer // here — but dbt does not trigger downstream runs. Its own DAG is dbt's to // order; the only thing a cascade would add is waking Windmill scripts that - // read a mart, and nothing outside dbt can declare a `dbt://` write, so - // that edge exists in one direction only. Cascading from a project whose - // per-run selection can build any subset of itself needs a per-run write set - // to be correct, which is a design worth doing deliberately rather than - // inferring. Until then dbt materializes and reports; it does not dispatch. + // read a mart. Cascading from a project whose per-run selection can build any + // subset of itself needs a per-run write set to be correct, which is a design + // worth doing deliberately rather than inferring: the deploy-time write set is + // not what ran, and the per-relation state table keeps one row per relation. + // Until then dbt materializes and reports; it does not dispatch. The opposite + // direction does: a native `// materialize manual dbt://…` script reaches the + // fan-out below on the ordinary path, on the strength of its own asset rows. if job.script_lang == Some(ScriptLang::Dbt) { return Ok(DispatchResult::default()); } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c7c119a5a7..b6380c02d0 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -2065,10 +2065,35 @@ fn apply_completed_job_cloud_usage( queued_job: &MiniCompletedJob, _duration: i64, ) { - if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 { + if !queued_job.is_flow() { + meter_execution_seconds( + db, + &queued_job.workspace_id, + &queued_job.permissioned_as_email, + _duration, + ); + } +} + +/// Charge `_duration` of execution time to the cloud usage meters: the workspace's +/// monthly row, plus the per-user row on non-premium plans. +/// +/// The unit is one finished **segment**, not one job. A Workflow-as-Code parent parks on +/// a sleep, an approval or its children and resumes with a fresh timer, so its compute +/// arrives here as several calls; metering only the one at completion would drop +/// everything it ran before its first park. +/// +/// Fire-and-forget, like every other write to `usage`: billing must never hold up the +/// job that produced it. +/// +/// `w_id` and `email` are billed as given and authorize nothing on their own — take them +/// from a job the caller already holds, never from request input. +#[cfg(feature = "cloud")] +pub fn meter_execution_seconds(db: &Pool, w_id: &str, email: &str, _duration: i64) { + if *CLOUD_HOSTED && _duration > 1000 { let db = db.clone(); - let w_id = queued_job.workspace_id.clone(); - let email = queued_job.permissioned_as_email.clone(); + let w_id = w_id.to_string(); + let email = email.to_string(); let w_id2 = w_id.clone(); let email2 = email.clone(); tokio::task::spawn(async move { @@ -4007,33 +4032,10 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { { let maybe_new_id = match j.kind { - JobKind::Dependencies => { - let deployment_message = j - .args - .clone() - .map(|hashmap| { - hashmap - .get("deployment_message") - .map(|map_value| serde_json::from_str::(map_value.get()).ok()) - .flatten() - }) - .flatten(); - - // This way we tell downstream which script we should archive when the resolution is finished. - // (not used at the moment) - j.args - .as_mut() - .map(|args| args.insert("base_hash".to_owned(), to_raw_value(&*base_hash))); - - windmill_common::scripts::clone_script( - j.runnable_path(), - &j.workspace_id, - deployment_message, - db, - ) - .await? - .new_hash - } + // A script gets its new version from the worker, once the generated lock is known + // to differ from the live version's: minting one here would deploy, and walk the + // importers of, a version whose lock turns out byte-identical to its parent's. + JobKind::Dependencies => *base_hash, JobKind::FlowDependencies => { sqlx::query_scalar!( "INSERT INTO flow_version @@ -5503,6 +5505,8 @@ async fn push_inner<'c, 'd>( ) { // Check current usage with SELECT (fast, no row locks) // Only check user usage for non-premium workspaces + // `email` here and in the per-user checks below can be a cached dispatch address, up + // to one notify poll stale; accepted, see `get_email_from_permissioned_as`. let (current_workspace_usage, current_user_usage) = check_usage_limits(db, &billing_w_id, email, !team_plan_status.premium).await?; @@ -6892,7 +6896,11 @@ async fn push_inner<'c, 'd>( language as Option, same_worker, pre_run_error.map(|e| e.to_string()), - email, + // `job_authed`'s, not the handed-in `email`: unless the caller's own authed already names + // this identity, it came through `fetch_authed_from_permissioned_as`, which re-resolves the + // address from the principal's live binding. The same statement writes it to + // `job_perms.email`, and the two columns naming different accounts is what this prevents. + job_authed.email, visible_to_owner, flow_innermost_root_job, guarded_concurrent_limit, @@ -7009,7 +7017,8 @@ async fn push_inner<'c, 'd>( hm.insert("created_by", user); } let audit_author = AuditAuthor { - email: email.to_string(), + // `job_authed`'s address, matching `v2_job` and `job_perms` above. + email: job_authed.email.clone(), username: if runs_on_behalf { windmill_common::auth::permissioned_as_to_username(&permissioned_as) } else { @@ -7306,15 +7315,19 @@ async fn check_workspace_queue_cap<'c>( // Ok(()) // } -pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value { - let reason = job - .canceled_reason - .as_deref() - .unwrap_or_else(|| "no reason given"); - let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown"); +/// The result payload a job cancelled anywhere carries. Callers that hold the cancel +/// outside a `MiniPulledJob` — a row read after the pull, say — go through this rather +/// than rebuilding the shape. +pub fn canceled_result(reason: Option<&str>, canceler: Option<&str>) -> serde_json::Value { + let reason = reason.unwrap_or("no reason given"); + let canceler = canceler.unwrap_or("unknown"); serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler}) } +pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value { + canceled_result(job.canceled_reason.as_deref(), job.canceled_by.as_deref()) +} + /// Helper function to create a restarted module for branch/iteration restart fn create_restarted_module( module: &FlowStatusModule, diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index 6e80b1405e..d44213d3bc 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -26,7 +26,7 @@ use std::{ cell::RefCell, path::PathBuf, rc::Rc, - sync::{Arc, Mutex}, + sync::{Arc, LazyLock, Mutex}, }; // Re-export deno_telemetry for use by windmill-worker's otel proxy @@ -182,10 +182,33 @@ struct LogString { pub s: mpsc::UnboundedSender, } -#[derive(Clone)] +#[derive(Clone, Default)] pub struct NativeAnnotation { pub useragent: Option, pub proxy: Option<(String, Option<(String, String)>)>, + /// `//fetch_response_timeout `: per-script override of + /// [`default_fetch_response_timeout_secs`]. `Some(0)` disables it for this + /// script; `None` leaves the default in force. + pub fetch_response_timeout_secs: Option, +} + +/// How long `fetch()` waits for a response to begin, in seconds; `0` disables. +/// +/// Covers everything up to the response headers and stops there, so a body may +/// then stream for any length of time. `src/runtime.js` holds the semantics. +/// +/// Must exceed `TIMEOUT_WAIT_RESULT` (default 600), which holds synchronous job +/// calls open without headers. Raising that hot-reloaded instance setting may +/// also require raising `WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS` in the deployment +/// and restarting workers: this environment value is cached for the process. +pub fn default_fetch_response_timeout_secs() -> u64 { + static SECS: LazyLock = LazyLock::new(|| { + std::env::var("WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS") + .ok() + .and_then(|x| x.trim().parse::().ok()) + .unwrap_or(900) + }); + *SECS } /// Serializes V8 isolate creation as defense-in-depth against concurrent @@ -401,7 +424,7 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { } pub fn get_annotation(inner_content: &str) -> NativeAnnotation { - let mut res = NativeAnnotation { useragent: None, proxy: None }; + let mut res = NativeAnnotation::default(); let anns = inner_content .lines() @@ -414,6 +437,13 @@ pub fn get_annotation(inner_content: &str) -> NativeAnnotation { res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string()); } else if ann.starts_with("proxy") { res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim()); + } else if ann.starts_with("fetch_response_timeout") { + // A typo falls back to the default, never to "no timeout". + res.fetch_response_timeout_secs = ann + .trim_start_matches("fetch_response_timeout") + .trim() + .parse::() + .ok(); } } res @@ -554,6 +584,16 @@ pub(crate) fn create_nativets_runtime( let ops = vec![op_get_static_args(), op_log()]; let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; + // deno_web's setTimeout puts its delay through `webidl.converters.long`, + // which wraps at 32 bits: past i32::MAX ms (~24.8 days) the delay comes out + // negative and fires immediately, so an over-generous setting would abort + // every fetch on the spot. Cap rather than wrap. + let fetch_response_timeout_ms = ann + .fetch_response_timeout_secs + .unwrap_or_else(default_fetch_response_timeout_secs) + .saturating_mul(1000) + .min(i32::MAX as u64); + let fetch_options = deno_fetch::Options { root_cert_store_provider: NATIVE_ROOT_CERT_STORE_PROVIDER.clone(), user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), @@ -625,10 +665,15 @@ pub(crate) fn create_nativets_runtime( } // Per-isolate JS init that can't run in the snapshot (runtime.js executes at - // snapshot-build time): currently seeds performance.timeOrigin via - // setTimeOrigin(), which must read this isolate's wall clock. + // snapshot-build time): the wall clock behind performance.timeOrigin and the + // fetch response timeout are both per-isolate values. js_runtime - .execute_script("", "globalThis.__wmInitPerIsolate()") + .execute_script( + "", + format!( + "globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms} }})" + ), + ) .map_err(windmill_common::error::to_anyhow)?; Ok(CreatedRuntime { js_runtime, log_receiver, memory_limit_rx }) diff --git a/backend/windmill-runtime-nativets/src/runtime.js b/backend/windmill-runtime-nativets/src/runtime.js index 01bec09983..88e10f765a 100644 --- a/backend/windmill-runtime-nativets/src/runtime.js +++ b/backend/windmill-runtime-nativets/src/runtime.js @@ -30,9 +30,115 @@ import * as performance from "ext:deno_web/15_performance.js"; import "ext:deno_web/16_image_data.js"; import "ext:deno_fetch/27_eventsource.js"; +// deno_fetch applies no deadline, so a peer that accepts a request and then +// never answers leaves `await fetch(...)` pending until the job timeout, which +// self-hosted defaults to 7 days. +const ORIGINAL_FETCH = fetch.fetch; + +// deno_web's timers reject any `this` other than undefined/globalThis, so +// `timers.setTimeout(...)` passes the module namespace and throws "Illegal +// invocation". +const setTimeoutUnbound = timers.setTimeout; +const clearTimeoutUnbound = timers.clearTimeout; +// Captured before user code shares the isolate and could redefine them, the +// way deno's own modules reach intrinsics through primordials. +const PromiseReject = Promise.reject.bind(Promise); +const promiseThen = Function.prototype.call.bind(Promise.prototype.then); +const abortSignalAny = abortSignal.AbortSignal.any.bind(abortSignal.AbortSignal); +const abortControllerAbort = Function.prototype.call.bind( + abortSignal.AbortController.prototype.abort, +); +const ReflectApply = Reflect.apply; + +// Installed per isolate by __wmInitPerIsolate; 0 disables. Only a backstop +// for the impossible case of fetch running before that init. +let fetchResponseTimeoutMs = 900_000; + +function fetchResponseTimeoutError(requestUrl, timeoutMs) { + let target; + try { + const parsed = new url.URL(requestUrl); + // Query and fragment routinely carry tokens, and this reaches a job log. + target = parsed.origin + parsed.pathname; + } catch { + target = "the request target"; + } + return new domException.DOMException( + `fetch to ${target} timed out: no response headers arrived within ` + + `${Math.round(timeoutMs / 1000)}s (this covers connect, request upload ` + + `and the wait for the server to start replying; once a response begins ` + + `it is never interrupted). Change it per script with ` + + `"//fetch_response_timeout " (0 disables), or instance-wide ` + + `with WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS.`, + "TimeoutError", + ); +} + globalThis.atob = base64.atob; globalThis.btoa = base64.btoa; -globalThis.fetch = fetch.fetch; +// Not `async`, for the same reason deno_fetch's own outer fetch isn't: WPT +// pins that an aborted fetch settles in the same tick, which adopting its +// promise through another one would break. Construction still has to reject +// rather than throw, so it is caught and handed back as a rejection. +globalThis.fetch = function fetch(input, init = undefined) { + const timeoutMs = fetchResponseTimeoutMs; + // Forwarded with the original argument count, so deno still sees an empty + // call as empty and raises its own "1 argument required". The default on + // `init` is what keeps `fetch.length` at 1, as the standard has it. + if (!(timeoutMs > 0) || arguments.length < 1) { + return ReflectApply(ORIGINAL_FETCH, undefined, arguments); + } + + let req; + let controller; + let signal; + try { + // RequestInit is a WebIDL dictionary: copying it drops inherited and + // non-enumerable members, and inheriting from it runs accessors against the + // wrong receiver. Hand it to the same Request constructor fetch() would, + // and carry our own signal in an init of our own. + req = new request.Request(input, init); + + // `req.signal` is deno's own resolution of init.signal over an input + // Request's signal, so combining with it preserves the caller's abort and + // reason while ours only adds a ceiling. + controller = new abortSignal.AbortController(); + signal = abortSignalAny([req.signal, controller.signal]); + } catch (e) { + return PromiseReject(e); + } + + // Already aborted: hand back deno's own settled rejection untouched, and arm + // nothing -- there is no response to wait for. + if (signal.aborted) { + return ORIGINAL_FETCH(req, { signal }); + } + + let timer = setTimeoutUnbound(() => { + timer = undefined; + abortControllerAbort(controller, fetchResponseTimeoutError(req.url, timeoutMs)); + }, timeoutMs); + // Disarmed on headers, never on body completion: a response that has begun + // arriving must be free to stream for as long as it needs. + const disarm = () => { + if (timer !== undefined) { + clearTimeoutUnbound(timer); + timer = undefined; + } + }; + + return promiseThen( + ORIGINAL_FETCH(req, { signal }), + (res) => { + disarm(); + return res; + }, + (e) => { + disarm(); + throw e; + }, + ); +}; globalThis.Request = request.Request; globalThis.Response = response.Response; globalThis.Blob = file.Blob; @@ -123,7 +229,11 @@ Object.assign(globalThis, { // Per-isolate init, invoked from Rust after the snapshot is restored (this // module body runs at snapshot-build time, not per isolate). -globalThis.__wmInitPerIsolate = () => { +globalThis.__wmInitPerIsolate = (config) => { + if (config != null && typeof config.fetchResponseTimeoutMs === "number") { + fetchResponseTimeoutMs = config.fetchResponseTimeoutMs; + } + // setTimeOrigin() seeds performance.timeOrigin from the isolate's wall clock; // without it timeOrigin is undefined and `timeOrigin + performance.now()` is NaN. performance.setTimeOrigin(); diff --git a/backend/windmill-runtime-nativets/src/smoke_tests.rs b/backend/windmill-runtime-nativets/src/smoke_tests.rs index b581a9f1fe..f3da460227 100644 --- a/backend/windmill-runtime-nativets/src/smoke_tests.rs +++ b/backend/windmill-runtime-nativets/src/smoke_tests.rs @@ -24,7 +24,7 @@ use crate::{transpile_ts, NativeAnnotation, PrewarmedIsolate, PrewarmedResult}; /// positional args, and return the isolate's result + captured logs. async fn run_ts(ts: &str, arg_names: &[&str], args: serde_json::Value) -> PrewarmedResult { let js = transpile_ts(ts.to_string()).expect("transpile_ts failed"); - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let arg_names: Vec = arg_names.iter().map(|s| s.to_string()).collect(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, arg_names, None); iso.wait_ready().await.expect("isolate failed to pre-warm"); @@ -232,7 +232,7 @@ export async function main(i: number): Promise { for i in 0..N { let js = js.clone(); let h = tokio::spawn(async move { - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec!["i".to_string()], None); iso.wait_ready().await.expect("pre-warm failed"); diff --git a/backend/windmill-runtime-nativets/tests/annotation.rs b/backend/windmill-runtime-nativets/tests/annotation.rs new file mode 100644 index 0000000000..d8319a87a5 --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/annotation.rs @@ -0,0 +1,37 @@ +//! `//fetch_response_timeout` parsing. Cheap (no V8), so it stays out of the +//! e2e file. + +use windmill_runtime_nativets::get_annotation; + +#[test] +fn a_value_is_parsed_and_zero_stays_distinct_from_absent() { + // Collapsing `Some(0)` to `None` would silently reinstate the default on a + // script that explicitly asked for no limit. + assert_eq!( + get_annotation("//native\n//fetch_response_timeout 30\n").fetch_response_timeout_secs, + Some(30) + ); + assert_eq!( + get_annotation("//fetch_response_timeout 0\n").fetch_response_timeout_secs, + Some(0) + ); + assert_eq!( + get_annotation("//native\n").fetch_response_timeout_secs, + None + ); +} + +#[test] +fn a_malformed_value_falls_back_to_the_default_not_to_no_timeout() { + for src in [ + "//fetch_response_timeout abc\n", + "//fetch_response_timeout\n", + "//fetch_response_timeout -5\n", + ] { + assert_eq!( + get_annotation(src).fetch_response_timeout_secs, + None, + "{src:?} should leave the default in force" + ); + } +} diff --git a/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs b/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs new file mode 100644 index 0000000000..5118b5f93f --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs @@ -0,0 +1,593 @@ +//! The nativets `fetch()` response timeout. +//! +//! Two halves of one contract, and a fix that satisfies only the first is worse +//! than no fix: +//! +//! 1. a peer that accepts a request and never answers is given up on +//! 2. a response that has begun arriving is never cut off, however long it +//! takes in total +//! +//! (2) rules out the obvious implementation — `AbortSignal.timeout(N)` around +//! every fetch would satisfy (1) and break every streaming response and long +//! download. +//! +//! Hermetic: loopback listeners, no egress. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::Mutex; + +use windmill_runtime_nativets::{transpile_ts, NativeAnnotation, PrewarmedIsolate}; + +/// The annotation is in whole seconds, so the tests scale around this. +const TIMEOUT_SECS: u64 = 2; + +async fn run_with_timeout_secs(ts: &str, secs: u64) -> Result { + let js = transpile_ts(ts.to_string()).expect("transpile_ts failed"); + let ann = + NativeAnnotation { fetch_response_timeout_secs: Some(secs), ..NativeAnnotation::default() }; + let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None); + iso.wait_ready().await.expect("isolate failed to pre-warm"); + let res = iso + .start_execution("{}".to_string()) + .wait() + .await + .expect("isolate panicked"); + res.result.map(|raw| raw.get().to_string()) +} + +/// A peer that reads the request and then answers nothing, closing only after +/// `close_after` so no test leaves an isolate wedged on a pending fetch. +/// +/// The socket is held rather than dropped on accept: dropping it sends a FIN, +/// which surfaces as a connection error — the easy failure, not this one. +async fn spawn_silent_peer(seen: Arc>>, close_after: Duration) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let seen = seen.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + if let Ok(n) = sock.read(&mut buf).await { + seen.lock().await.extend_from_slice(&buf[..n]); + } + tokio::time::sleep(close_after).await; + drop(sock); + }); + } + }); + port +} + +/// A peer that records the request it received and answers 200 immediately. +async fn spawn_echo_peer(seen: Arc>>) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let seen = seen.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + if let Ok(n) = sock.read(&mut buf).await { + seen.lock().await.extend_from_slice(&buf[..n]); + } + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + }); + } + }); + port +} + +/// A peer that responds after `headers_after`, then dribbles a chunked body out +/// over `chunks * chunk_every`. +async fn spawn_streaming_peer( + headers_after: Duration, + chunks: usize, + chunk_every: Duration, +) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + let _ = sock.read(&mut buf).await; + tokio::time::sleep(headers_after).await; + if sock + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n", + ) + .await + .is_err() + { + return; + } + for _ in 0..chunks { + tokio::time::sleep(chunk_every).await; + if sock.write_all(b"1\r\nx\r\n").await.is_err() { + return; + } + } + let _ = sock.write_all(b"0\r\n\r\n").await; + }); + } + }); + port +} + +const POST_TO_SILENT_PEER: &str = r#" +export async function main(): Promise { + const res = await fetch("http://127.0.0.1:{port}/orders", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: 1 }), + }); + return res.status; +} +"#; + +fn post_script(port: u16) -> String { + POST_TO_SILENT_PEER.replace("{port}", &port.to_string()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_peer_that_never_answers_is_given_up_on() { + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen.clone(), Duration::from_secs(60)).await; + + let started = Instant::now(); + let err = run_with_timeout_secs(&post_script(port), TIMEOUT_SECS) + .await + .expect_err("fetch against a peer that never answers must not resolve"); + let elapsed = started.elapsed(); + + assert!( + elapsed >= Duration::from_secs(TIMEOUT_SECS), + "gave up after {elapsed:?}, before the configured {TIMEOUT_SECS}s -- \ + the timeout is firing on something other than the wait for a response", + ); + assert!( + elapsed < Duration::from_secs(TIMEOUT_SECS + 15), + "took {elapsed:?} to give up", + ); + + // The message has to stand on its own in a job log: a hung request is + // otherwise indistinguishable from a slow one. + assert!( + err.contains("no response headers arrived within"), + "error should explain what timed out, got: {err}", + ); + assert!( + err.contains("/orders") && err.contains("fetch_response_timeout"), + "error should name the target and how to change the limit, got: {err}", + ); + + // Without this the test would also pass if the request never went out. + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.contains("POST /orders"), + "peer should have received the request, got: {body}", + ); +} + +/// The counterfactual for the test above: with the timeout disabled, the same +/// script against the same peer is still running well past the point the +/// timeout would have fired. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn without_a_timeout_the_same_request_keeps_running() { + let seen = Arc::new(Mutex::new(Vec::new())); + // Closes eventually, so the isolate unwinds instead of pinning a blocking + // task for the rest of the test binary's life. + let port = spawn_silent_peer(seen, Duration::from_secs(TIMEOUT_SECS * 3)).await; + + let still_running = tokio::time::timeout( + Duration::from_secs(TIMEOUT_SECS * 2), + run_with_timeout_secs(&post_script(port), 0), + ) + .await + .is_err(); + + assert!( + still_running, + "with the timeout disabled the request should still have been pending \ + at {}s -- if it ends on its own, the test above proves nothing", + TIMEOUT_SECS * 2, + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_streaming_body_outliving_the_timeout_is_not_cut_off() { + // Headers land fast, then the body trickles well past the timeout. A + // total-duration timeout fails here; that is the point of the test. + let port = + spawn_streaming_peer(Duration::from_millis(200), 10, Duration::from_millis(500)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/stream"); + return `${{res.status}}:${{(await res.text()).length}}`; +}} +"# + ); + + let started = Instant::now(); + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a streaming response must not be interrupted"); + let elapsed = started.elapsed(); + + assert_eq!(out, "\"200:10\"", "full body should arrive intact"); + assert!( + elapsed > Duration::from_secs(TIMEOUT_SECS), + "the transfer ({elapsed:?}) has to outlast the {TIMEOUT_SECS}s timeout \ + for this to be exercising anything", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_slow_but_answering_peer_is_not_cut_off() { + // A quarter of the window rather than half: CI runs this at + // --test-threads=10 alongside other V8 isolates, and this margin is what + // absorbs executor starvation. + let port = spawn_streaming_peer( + Duration::from_millis(1_000 * TIMEOUT_SECS / 4), + 1, + Duration::from_millis(10), + ) + .await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/slow"); + await res.text(); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a slow but answering peer must not be cut off"); + assert_eq!(out, "200"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_over_large_timeout_does_not_wrap_into_an_instant_one() { + // deno_web's setTimeout puts its delay through `webidl.converters.long`, + // which wraps at 32 bits. Unclamped, this ~46-day setting wraps negative and + // aborts immediately -- asking for a longer leash would kill every fetch. + let port = spawn_streaming_peer(Duration::from_millis(50), 1, Duration::from_millis(10)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/ok"); + await res.text(); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, 4_000_000) + .await + .expect("an over-large timeout must not abort the request"); + assert_eq!(out, "200"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_caller_abort_still_wins_with_its_own_reason() { + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(60)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const ac = new AbortController(); + setTimeout(() => ac.abort(new Error("caller_abort_marker")), 200); + try {{ + await fetch("http://127.0.0.1:{port}/probe", {{ signal: ac.signal }}); + return "unexpectedly resolved"; + }} catch (e) {{ + return String((e as Error).message); + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch its own abort"); + assert!( + out.contains("caller_abort_marker"), + "caller's abort reason should survive being combined with ours, got: {out}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_timeout_is_catchable_as_a_timeout_error() { + // Scripts that retry on transient failures need to recognise this one; + // `TimeoutError` matches AbortSignal.timeout()'s reason. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(60)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + try {{ + await fetch("http://127.0.0.1:{port}/probe"); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch the timeout"); + assert_eq!(out, "\"TimeoutError\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_init_whose_members_are_inherited_is_not_flattened() { + // RequestInit is a WebIDL dictionary and deno_fetch reads its members with + // plain property gets, so they may sit on the prototype chain or be + // non-enumerable. Object spread copies neither, which would silently + // downgrade this POST to a GET and drop the header. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +declare const Object: any; +export async function main(): Promise {{ + const base = {{ method: "POST", headers: {{ "x-probe": "yes" }} }}; + const res = await fetch("http://127.0.0.1:{port}/inherited", Object.create(base)); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("request should succeed"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("POST /inherited"), + "inherited `method` should survive, got: {body}", + ); + assert!( + body.to_lowercase().contains("x-probe: yes"), + "inherited `headers` should survive, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_dictionary_init_still_fails_loudly() { + // deno's dictionary converter throws on a non-object init. Copying members + // into a fresh object instead of inheriting would turn `"POST"` into + // {0:"P",1:"O",...} -- a valid dictionary with ignored keys, i.e. a silent + // GET where the caller used to get a TypeError. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + try {{ + await fetch("http://127.0.0.1:{port}/x", "POST" as any); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch the error"); + assert_eq!(out, "\"TypeError\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_accessor_backed_init_reads_against_its_own_receiver() { + // A getter on the init must run with the object it was defined on as `this`, + // or a private field is unreachable and it throws. Carrying the init across + // by inheritance rather than by handing it to Request would break this. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +class Init {{ + #method = "POST"; + #body = "from-private-field"; + get method(): string {{ return this.#method; }} + get body(): string {{ return this.#body; }} +}} +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/accessor", new Init() as any); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("an accessor-backed init must not throw"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("POST /accessor") && body.contains("from-private-field"), + "getter-provided method and body should both reach the wire, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_request_input_keeps_its_body_and_headers() { + // The wrapper builds a Request and hands that to fetch, so the body survives + // one more construction than it used to -- deno proxies it rather than + // consuming it, and this pins that. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const req = new Request("http://127.0.0.1:{port}/from-request", {{ + method: "PUT", + headers: {{ "x-probe": "yes" }}, + body: "payload-body", + }}); + const res = await fetch(req); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a Request input must work"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("PUT /from-request") + && body.to_lowercase().contains("x-probe: yes") + && body.contains("payload-body"), + "method, headers and body should all survive, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_already_aborted_fetch_settles_in_the_same_tick() { + // deno_fetch keeps its outer fetch non-async and returns an already-settled + // rejection untouched, because WPT pins that an aborted fetch settles in the + // same tick. Adopting it through another promise pushes the rejection behind + // any microtask queued after the call. + let ts = r#" +export async function main(): Promise { + const order: string[] = []; + const ac = new AbortController(); + ac.abort(); + const f = fetch("http://127.0.0.1:1/x", { signal: ac.signal }) + .catch(() => { order.push("fetch"); }); + Promise.resolve().then(() => { order.push("queued-after"); }); + await f; + await new Promise((r) => setTimeout(r, 0)); + return order.join(","); +} +"#; + + let out = run_with_timeout_secs(ts, TIMEOUT_SECS) + .await + .expect("script should run"); + assert_eq!( + out, "\"fetch,queued-after\"", + "the rejection must land before a microtask queued after the call", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_wrapper_keeps_fetch_s_own_shape() { + // The wrapper is indistinguishable from deno's fetch on three counts a + // script can observe: its arity, the error for an empty call, and not + // depending on a mutable `Promise.prototype.then` the way an ordinary + // property lookup would. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen).await; + + let ts = format!( + r#" +declare const Promise: any; +declare const AbortSignal: any; +declare const AbortController: any; +export async function main(): Promise {{ + const arity = (fetch as any).length; + + // The message, not just the type: forwarding two explicit `undefined`s + // would still throw a TypeError, just deno's invalid-URL one instead of + // its required-argument one. + let emptyCall = "resolved"; + try {{ + await (fetch as any)(); + }} catch (e) {{ + emptyCall = (e as Error).message.includes("1 argument required") + ? "required-argument" + : `other(${{(e as Error).message}})`; + }} + + // Patched only across the call: the wrapper reaches for these while + // building its return value, and awaiting under a patched Promise + // prototype would instead measure V8 treating it as a plain thenable. + const originalThen = Promise.prototype.then; + const originalAny = AbortSignal.any; + const originalAbort = AbortController.prototype.abort; + let pending: any; + try {{ + Promise.prototype.then = undefined; + (AbortSignal as any).any = undefined; + (AbortController.prototype as any).abort = undefined; + pending = fetch("http://127.0.0.1:{port}/shape"); + }} finally {{ + Promise.prototype.then = originalThen; + (AbortSignal as any).any = originalAny; + (AbortController.prototype as any).abort = originalAbort; + }} + const status = (await pending).status; + + return `${{arity}}:${{emptyCall}}:${{status}}`; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should run"); + assert_eq!(out, "\"1:required-argument:200\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_timer_aborts_through_a_captured_intrinsic() { + // The timeout has to fire for this one: `AbortController.prototype.abort` + // is patched out for the whole wait, so an ordinary lookup would throw + // inside the timer callback and leave the request pending forever. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(TIMEOUT_SECS * 5)).await; + + let ts = format!( + r#" +declare const AbortController: any; +export async function main(): Promise {{ + const originalAbort = AbortController.prototype.abort; + AbortController.prototype.abort = undefined; + try {{ + await fetch("http://127.0.0.1:{port}/patched-abort"); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} finally {{ + AbortController.prototype.abort = originalAbort; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("the timeout must still fire"); + assert_eq!(out, "\"TimeoutError\""); +} diff --git a/backend/windmill-runtime-nativets/tests/otel_e2e.rs b/backend/windmill-runtime-nativets/tests/otel_e2e.rs index cadddb2fcf..22fd18de90 100644 --- a/backend/windmill-runtime-nativets/tests/otel_e2e.rs +++ b/backend/windmill-runtime-nativets/tests/otel_e2e.rs @@ -136,7 +136,7 @@ export async function main(): Promise {{ "# ); let js = transpile_ts(ts).expect("transpile failed"); - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None); iso.wait_ready().await.expect("isolate failed to pre-warm"); diff --git a/backend/windmill-runtime-nativets/tests/response_timeout_env.rs b/backend/windmill-runtime-nativets/tests/response_timeout_env.rs new file mode 100644 index 0000000000..a206c63134 --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/response_timeout_env.rs @@ -0,0 +1,13 @@ +//! Its own test binary: the setting is read once per process through a +//! `LazyLock`, so nothing else may resolve it first. Adding a second test to +//! this file breaks that isolation. + +use windmill_runtime_nativets::default_fetch_response_timeout_secs; + +#[test] +fn the_env_var_is_what_operators_actually_set() { + // A typo in the variable's name would compile, pass every other test, and + // silently hand every operator the built-in default instead. + std::env::set_var("WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS", "17"); + assert_eq!(default_fetch_response_timeout_secs(), 17); +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index f9bfe2f4df..126cf97106 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -7,7 +7,7 @@ */ use dashmap::DashMap; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::IpAddr; use std::sync::LazyLock; @@ -17,6 +17,7 @@ use windmill_api_auth::{ }; use windmill_common::db::DB; use windmill_common::per_minute_counter::PerMinuteCounter; +use windmill_common::ssrf::{private_git_host_allowed, private_git_host_hint, GitRemoteCaller}; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; @@ -50,9 +51,9 @@ use windmill_common::{ error::{self, Error, JsonResult, Result}, get_database_url, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only, - fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, WithDraftOverlay, - WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only, fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, + WithDraftOverlay, WithDraftQuery, }, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, @@ -89,6 +90,9 @@ pub fn workspaced_service() -> Router { .route("/git_commit_hash/{*path}", get(get_git_commit_hash)) .route("/type/list", get(list_resource_types)) .route("/type/listnames", get(list_resource_types_names)) + .route("/type/resource_counts", get(list_resource_counts_by_type)) + .route("/type/hub/info", get(list_hub_resource_type_info)) + .route("/type/hub/pick/{name}", post(pick_hub_resource_type)) .route("/type/get/{name}", get(get_resource_type)) .route("/type/exists/{name}", get(exists_resource_type)) .route("/type/update/{name}", post(update_resource_type)) @@ -134,7 +138,10 @@ pub struct EditResourceType { /// `Option` conflates: an absent field leaves the extension alone, while an /// explicit `null` clears it. A hub pull relies on both — a type that stops /// being a file type has to stop being one locally too. - #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] pub format_extension: Option>, } @@ -1290,9 +1297,26 @@ async fn create_resource( webhook.send_message( w_id.clone(), - WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() }, + WebhookMessage::CreateResource { workspace: w_id.clone(), path: resource.path.clone() }, ); + // Trigger CI tests for items that reference this resource + { + let db2 = db.clone(); + let path2 = resource.path.clone(); + let email2 = authed.email.clone(); + let username2 = authed.username.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_dep_map::ci_tests::trigger_ci_tests_for_item( + &db2, &w_id, &path2, "resource", &email2, &username2, + ) + .await + { + tracing::error!(%e, "error triggering CI tests after resource creation"); + } + }); + } + Ok(( StatusCode::CREATED, format!("resource {} created", resource.path), @@ -1309,6 +1333,17 @@ async fn delete_resource( let path = path.to_path(); check_scopes(&authed, || format!("resources:write:{}", path))?; + + // Ahead of the deploy rules: nothing is deployed at a draft-only path, so + // gating this discard on them would strand the row in a protected workspace. + // Ahead of the transaction too — the not-found branch other kinds hang this + // off is the `not_found_if_none` below, past the linked-variable cascade. + if delete_draft_only_for_path(&db, &w_id, UserDraftItemKind::Resource, path, &authed.email) + .await? + { + return Ok(format!("draft-only resource {} deleted", path)); + } + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -1320,53 +1355,22 @@ async fn delete_resource( { return Err(Error::PermissionDenied(msg)); } + + let cascade = plan_linked_var_cascade(&db, &w_id, &[path.to_string()]).await?; + let mut tx = user_db.begin(&authed).await?; - // Capture resource data for trashbin before deleting - let trash_resource: Option = sqlx::query_scalar( - "SELECT to_jsonb(t) FROM resource t WHERE path = $1 AND workspace_id = $2", + // The whole row comes back out of the delete, so the trashbin entry below is built from + // what RLS actually removed, and the cascade runs only once RLS has allowed the delete. + let deleted: Option<(String, serde_json::Value)> = sqlx::query_as( + "DELETE FROM resource AS t WHERE t.path = $1 AND t.workspace_id = $2 + RETURNING t.path, to_jsonb(t)", ) .bind(path) .bind(&w_id) .fetch_optional(&mut *tx) .await?; - - // Fetch the resource value before deleting, so we can find linked $var: references - let resource_value: Option> = - sqlx::query_scalar("SELECT value FROM resource WHERE path = $1 AND workspace_id = $2") - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - // Collect all $var: paths referenced in the resource value - let mut linked_var_paths: Vec = Vec::new(); - if let Some(Some(ref value)) = resource_value { - collect_var_refs(value, &mut linked_var_paths); - } - - // A scoped token must not delete linked variables it lacks variables:write for. - check_linked_var_delete_scopes(&authed, &linked_var_paths)?; - - // Capture linked variables for trashbin before deleting them - let trash_linked_vars: Vec = if linked_var_paths.is_empty() { - Vec::new() - } else { - let placeholders: Vec = linked_var_paths - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "SELECT to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path IN ({})", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, serde_json::Value>(&query).bind(&w_id); - for var_path in &linked_var_paths { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; + let (deleted_path, res_data) = not_found_if_none(deleted, "Resource", &path)?; sqlx::query!( "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2", @@ -1376,64 +1380,63 @@ async fn delete_resource( .execute(&mut *tx) .await?; - let deleted_path = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", - path, - w_id + let linked_var_paths = cascade.resolve(std::slice::from_ref(&deleted_path)); + + // A scoped token must not delete linked variables it lacks variables:write for. Erroring + // here rolls the resource delete back with it, so nothing is deleted either way. + check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + + // Capture linked variables for trashbin before deleting them + let trash_linked_vars: Vec<(String, serde_json::Value)> = sqlx::query_as( + "SELECT path, to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path = ANY($2)", ) - .fetch_optional(&mut *tx) + .bind(&w_id) + .bind(&linked_var_paths) + .fetch_all(&mut *tx) .await?; - not_found_if_none(deleted_path, "Resource", &path)?; - // Delete linked variables that are actually referenced in the resource value - let deleted_linked_variables: Vec = if linked_var_paths.is_empty() { - Vec::new() - } else { - // Clean up any ws_specific rows for these variables first - // (mark_linked_variables_ws_specific may have auto-inserted them) so - // they don't survive the variable deletion as orphans — a variable - // later recreated at the same path would otherwise inherit the stale - // ws_specific flag. - sqlx::query!( - "DELETE FROM ws_specific - WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", - w_id, - &linked_var_paths - ) - .execute(&mut *tx) - .await?; + let deleted_linked_variables = sqlx::query_scalar!( + "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + w_id, + &linked_var_paths + ) + .fetch_all(&mut *tx) + .await?; - let placeholders: Vec = linked_var_paths - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "DELETE FROM variable WHERE workspace_id = $1 AND path IN ({}) RETURNING path", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, String>(&query).bind(&w_id); - for var_path in &linked_var_paths { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; + // ws_specific has no FK to variable, so a row mark_linked_variables_ws_specific inserted + // would survive as an orphan and a variable later recreated at that path would inherit + // the stale flag. + sqlx::query!( + "DELETE FROM ws_specific + WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + w_id, + &deleted_linked_variables + ) + .execute(&mut *tx) + .await?; - if let Some(res_data) = trash_resource { - let mut trash_data = serde_json::json!({"row": res_data}); - if !trash_linked_vars.is_empty() { - trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); - } - windmill_common::trashbin::move_to_trash( - &mut *tx, - &w_id, - "resource", - path, - trash_data, - &authed.username, - ) - .await?; + // Only the rows that actually went: the snapshot above is what the caller could read, + // which is not necessarily what RLS let it delete, and the trashbin must not hold a copy + // of a secret that is still live. + let trash_linked_vars: Vec = trash_linked_vars + .into_iter() + .filter(|(var_path, _)| deleted_linked_variables.contains(var_path)) + .map(|(_, row)| row) + .collect(); + + let mut trash_data = serde_json::json!({"row": res_data}); + if !trash_linked_vars.is_empty() { + trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); } + windmill_common::trashbin::move_to_trash( + &mut *tx, + &w_id, + "resource", + path, + trash_data, + &authed.username, + ) + .await?; audit_log( &mut *tx, @@ -1445,6 +1448,24 @@ async fn delete_resource( None, ) .await?; + + // The cascade is the one way a variable dies without a variables/delete request of its + // own, so give each one the audit row it would have had, stamped with what took it. + for var_path in &deleted_linked_variables { + let mut params = HashMap::new(); + params.insert("via_resource", path); + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(var_path), + Some(params), + ) + .await?; + } + tx.commit().await?; // Resource gone for everyone: wipe ALL users' drafts at this path (and any linked @@ -1496,35 +1517,205 @@ async fn delete_resource( ); } - Ok(format!("resource {} deleted", path)) + // Name what else went: the cascade is silent from the caller's side otherwise, and a + // secret it took is not something to discover later from a failing job. + if deleted_linked_variables.is_empty() { + Ok(format!("resource {} deleted", path)) + } else { + Ok(format!( + "resource {} deleted, along with its linked variables: {}", + path, + deleted_linked_variables.join(", ") + )) + } } -/// Recursively collect all `$var:path` references from a JSON value. -fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { +/// The forms that resolve a variable path against `variable`, so a value carrying any of them +/// breaks when that variable goes. Only `$var:` is minted by the resource editor, which is why +/// `collect_var_refs` stays narrower than this. +const REFERRER_PREFIXES: [&str; 2] = ["$var:", "$jsonvar:"]; + +/// Recursively collect the variable paths a JSON value references through any of `prefixes`. +fn collect_refs_with_prefixes(value: &serde_json::Value, prefixes: &[&str], out: &mut Vec) { match value { serde_json::Value::String(s) => { - if let Some(var_path) = s.strip_prefix("$var:") { + if let Some(var_path) = prefixes.iter().find_map(|p| s.strip_prefix(p)) { out.push(var_path.to_string()); } } serde_json::Value::Object(m) => { for v in m.values() { - collect_var_refs(v, out); + collect_refs_with_prefixes(v, prefixes, out); } } serde_json::Value::Array(arr) => { for v in arr { - collect_var_refs(v, out); + collect_refs_with_prefixes(v, prefixes, out); } } _ => {} } } -/// Deleting a resource cascades into the `$var:` variables its value references. A -/// scoped token must not use that cascade to delete variables it could not delete -/// directly via `delete_variable` (which gates on `variables:write:`), so require -/// `variables:write` for EVERY linked variable and fail the whole delete otherwise. +/// Recursively collect all `$var:path` references from a JSON value. +fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { + collect_refs_with_prefixes(value, &["$var:"], out) +} + +/// Whether the variable at `var_path` is the resource's own secret rather than one its value +/// merely points at: the same-path twin `delete_variable` and the `update_resource` rename +/// already act on, or `_`, which the connect form mints for a resource +/// type with several secret fields. Anything else is a standalone workspace variable, and +/// deleting one destroys a secret its other referrers still need. +/// +/// A rename moves only the twin, so `_` secrets stop matching and are left +/// behind instead. An orphaned secret can be deleted by hand; a destroyed one cannot. +fn is_owned_linked_var(resource_path: &str, var_path: &str) -> bool { + var_path == resource_path + || var_path + .strip_prefix(resource_path) + .is_some_and(|suffix| suffix.starts_with('_')) +} + +/// Which of `var_paths` a resource outside `excluded_resource_paths` still references. +/// +/// Must run off the non-RLS pool: a referrer in a folder the caller cannot read is precisely +/// the one whose variable has to survive. Nothing from those rows reaches the response. +async fn linked_vars_referenced_elsewhere( + db: &DB, + w_id: &str, + var_paths: &[String], + excluded_resource_paths: &[String], +) -> Result> { + if var_paths.is_empty() { + return Ok(HashSet::new()); + } + // The quotes around the pattern are what make it a whole-JSON-string match rather than a + // prefix one, so `f/db` does not match `"$var:f/db_replica"`. Paths are `proper_id` + // segments, so no JSON escaping or LIKE wildcard can reach this. + let referenced = sqlx::query_scalar!( + "WITH survivors AS ( + SELECT value::text AS rendered FROM resource + WHERE workspace_id = $1 AND NOT (path = ANY($2::text[])) + ) + SELECT v.path FROM unnest($3::text[]) AS v(path) + WHERE EXISTS ( + SELECT 1 FROM survivors s + WHERE strpos(s.rendered, '\"$var:' || v.path || '\"') > 0 + OR strpos(s.rendered, '\"$jsonvar:' || v.path || '\"') > 0 + )", + w_id, + excluded_resource_paths, + var_paths, + ) + .fetch_all(db) + .await?; + Ok(referenced.into_iter().flatten().collect()) +} + +/// A resource delete's candidate cascade, gathered before the caller's transaction opens: the +/// referrer scan runs on `db` because it has to see resources RLS hides, and a second acquire +/// from that same pool under an open `user_db` transaction stalls to the acquire timeout when +/// `DATABASE_CONNECTIONS` is small. `resolve` then decides without touching the database. +/// +/// So a resource that starts referencing a candidate between the scan and the delete keeps a +/// `$var:` pointing at nothing. Narrowing that window means running the scan on the +/// transaction's own connection under a tightly scoped `SET LOCAL ROLE NONE` (the elevation +/// `windmill-queue/src/schedule.rs` uses); closing it needs a lock on every resource write. +struct LinkedVarCascade { + /// Each owned `$var:` path with the requested resource whose value carries it. + candidates: Vec<(String, String)>, + /// Requested resource paths, each with every variable path its value references. + requested: Vec<(String, Vec)>, + /// Candidate paths a resource outside the requested set still references. + referenced_outside: HashSet, +} + +impl LinkedVarCascade { + /// The variables to delete, now that RLS has settled which resources went. + /// + /// A requested resource left standing is the one referrer the scan could not account for, + /// having had to exclude every requested path before RLS had ruled. + fn resolve(&self, deleted_paths: &[String]) -> Vec { + let referenced_by_survivors: HashSet<&str> = self + .requested + .iter() + .filter(|(path, _)| !deleted_paths.contains(path)) + .flat_map(|(_, refs)| refs.iter().map(String::as_str)) + .collect(); + + let mut resolved: Vec = self + .candidates + .iter() + .filter(|(var_path, owner)| { + deleted_paths.contains(owner) + && !self.referenced_outside.contains(var_path.as_str()) + && !referenced_by_survivors.contains(var_path.as_str()) + }) + .map(|(var_path, _)| var_path.clone()) + .collect(); + resolved.sort(); + resolved.dedup(); + resolved + } + + /// Which deleted resource the cascade took `var_path` for, to stamp on its audit row. + fn owner_of<'a>(&'a self, var_path: &str, deleted_paths: &[String]) -> Option<&'a str> { + self.candidates + .iter() + .find(|(candidate, owner)| candidate == var_path && deleted_paths.contains(owner)) + .map(|(_, owner)| owner.as_str()) + } +} + +/// Gather what `LinkedVarCascade::resolve` needs for a delete of `paths`. +async fn plan_linked_var_cascade( + db: &DB, + w_id: &str, + paths: &[String], +) -> Result { + let rows: Vec<(String, Option)> = sqlx::query_as( + "SELECT path, value FROM resource WHERE workspace_id = $1 AND path = ANY($2)", + ) + .bind(w_id) + .bind(paths) + .fetch_all(db) + .await?; + + let mut candidates: Vec<(String, String)> = Vec::new(); + let mut requested: Vec<(String, Vec)> = Vec::new(); + for (path, value) in rows { + let mut owned: Vec = Vec::new(); + let mut refs: Vec = Vec::new(); + if let Some(value) = &value { + collect_var_refs(value, &mut owned); + collect_refs_with_prefixes(value, &REFERRER_PREFIXES, &mut refs); + } + candidates.extend( + owned + .into_iter() + .filter(|var_path| is_owned_linked_var(&path, var_path)) + .map(|var_path| (var_path, path.clone())), + ); + requested.push((path, refs)); + } + candidates.sort(); + candidates.dedup(); + + let mut candidate_paths: Vec = candidates + .iter() + .map(|(var_path, _)| var_path.clone()) + .collect(); + candidate_paths.dedup(); + let referenced_outside = + linked_vars_referenced_elsewhere(db, w_id, &candidate_paths, paths).await?; + Ok(LinkedVarCascade { candidates, requested, referenced_outside }) +} + +/// Deleting a resource cascades into the `$var:` variables it owns. A scoped token must not +/// use that cascade to delete variables it could not delete directly via `delete_variable` +/// (which gates on `variables:write:`), so require `variables:write` for EVERY cascaded +/// variable and fail the whole delete otherwise. /// /// No co-located-path exemption: a resource and a variable may share a path, and a /// resource-write token can create a resource over an existing standalone variable and @@ -1627,111 +1818,91 @@ async fn delete_resources_bulk( return Err(Error::PermissionDenied(msg)); } + let cascade = plan_linked_var_cascade(&db, &w_id, &request.paths).await?; + let mut tx = user_db.begin(&authed).await?; - // Capture resources for trashbin per path before bulk delete, and - // collect $var: references so we can cascade-delete the linked variables - // (matching single-resource delete semantics). - let mut linked_var_paths: Vec = Vec::new(); - for path in &request.paths { - let trash_resource: Option = sqlx::query_scalar( - "SELECT to_jsonb(t) FROM resource t WHERE path = $1 AND workspace_id = $2", - ) - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - if let Some(res_data) = trash_resource { - // Per-resource linked vars so each resource's trash entry carries - // exactly the variables that vanished with it (matching the - // single-delete shape: trash_data["linked_variables"]). - let mut this_linked: Vec = Vec::new(); - if let Some(value) = res_data.get("value") { - collect_var_refs(value, &mut this_linked); - } - this_linked.sort(); - this_linked.dedup(); - - let trash_linked_vars: Vec = if this_linked.is_empty() { - Vec::new() - } else { - let placeholders: Vec = this_linked - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "SELECT to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path IN ({})", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, serde_json::Value>(&query).bind(&w_id); - for var_path in &this_linked { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; - - let mut trash_data = serde_json::json!({"row": res_data}); - if !trash_linked_vars.is_empty() { - trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); - } - windmill_common::trashbin::move_to_trash( - &mut *tx, - &w_id, - "resource", - path, - trash_data, - &authed.username, - ) - .await?; - - linked_var_paths.extend(this_linked); - } - } - linked_var_paths.sort(); - linked_var_paths.dedup(); - - // A scoped token must not delete linked variables it lacks variables:write for. - check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + // Whole rows out of the delete; see delete_resource. RLS can leave a requested resource + // standing, so everything below is driven by this list rather than by `request.paths`. + let deleted: Vec<(String, serde_json::Value)> = sqlx::query_as( + "DELETE FROM resource AS t WHERE t.path = ANY($1) AND t.workspace_id = $2 + RETURNING t.path, to_jsonb(t)", + ) + .bind(&request.paths) + .bind(&w_id) + .fetch_all(&mut *tx) + .await?; + let deleted_paths: Vec = deleted.iter().map(|(path, _)| path.clone()).collect(); sqlx::query!( "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = ANY($2)", w_id, - &request.paths + &deleted_paths ) .execute(&mut *tx) .await?; - let deleted_paths = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", - &request.paths, - w_id + let linked_var_paths = cascade.resolve(&deleted_paths); + + // A scoped token must not delete linked variables it lacks variables:write for. Erroring + // here rolls the resource deletes back with it. + check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + + // Snapshot before the delete below: the trashbin entries need the rows. + let trash_linked_vars: Vec<(String, serde_json::Value)> = sqlx::query_as( + "SELECT path, to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path = ANY($2)", + ) + .bind(&w_id) + .bind(&linked_var_paths) + .fetch_all(&mut *tx) + .await?; + + let deleted_linked_variables = sqlx::query_scalar!( + "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + w_id, + &linked_var_paths ) .fetch_all(&mut *tx) .await?; - // Cascade-clean linked variables: delete any ws_specific 'variable' rows - // (typically auto-inserted by mark_linked_variables_ws_specific when the - // resource was ws_specific) BEFORE deleting the variable rows themselves - // — otherwise those ws_specific rows survive as orphans and a later - // variable created at the same path would inherit a stale flag. - if !linked_var_paths.is_empty() { - sqlx::query!( - "DELETE FROM ws_specific - WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", - w_id, - &linked_var_paths - ) - .execute(&mut *tx) - .await?; + // See delete_resource: ws_specific has no FK, so the rows would orphan. + sqlx::query!( + "DELETE FROM ws_specific + WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + w_id, + &deleted_linked_variables + ) + .execute(&mut *tx) + .await?; - sqlx::query!( - "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2)", - w_id, - &linked_var_paths + for (path, res_data) in &deleted { + // Every cascaded variable this resource's value points at, ownership aside: restoring + // it on its own must bring back each secret it needs, and the one it borrowed from a + // sibling in the same batch is gone too. + let mut refs: Vec = Vec::new(); + if let Some(value) = res_data.get("value") { + collect_var_refs(value, &mut refs); + } + let this_linked: Vec = trash_linked_vars + .iter() + .filter(|(var_path, _)| { + refs.contains(var_path) && deleted_linked_variables.contains(var_path) + }) + .map(|(_, row)| row.clone()) + .collect(); + + let mut trash_data = serde_json::json!({"row": res_data}); + if !this_linked.is_empty() { + trash_data["linked_variables"] = serde_json::Value::Array(this_linked); + } + windmill_common::trashbin::move_to_trash( + &mut *tx, + &w_id, + "resource", + path, + trash_data, + &authed.username, ) - .execute(&mut *tx) .await?; } @@ -1746,13 +1917,30 @@ async fn delete_resources_bulk( ) .await?; + // See delete_resource: a cascaded variable gets the audit row it would have had. + for var_path in &deleted_linked_variables { + let params = cascade + .owner_of(var_path, &deleted_paths) + .map(|resource_path| HashMap::from([("via_resource", resource_path)])); + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(var_path), + params, + ) + .await?; + } + tx.commit().await?; // Wipe ALL users' drafts at these paths (and linked variables); see delete_resource. for path in &deleted_paths { delete_all_drafts_for_path(&db, &w_id, UserDraftItemKind::Resource, path).await?; } - for var_path in &linked_var_paths { + for var_path in &deleted_linked_variables { delete_all_drafts_for_path(&db, &w_id, UserDraftItemKind::Variable, var_path).await?; } @@ -2566,6 +2754,352 @@ async fn list_resource_types_names( Ok(Json(rows)) } +#[derive(Serialize)] +struct ResourceTypeCount { + resource_type: String, + count: i64, +} + +/// How many resources of each type this workspace holds — how popular a type is *here*, +/// which is what the pickers rank on below the hub's own pick counts. +async fn list_resource_counts_by_type( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + // A count per type is aggregate, so there is no path to narrow it by: a token scoped + // to individual resources gets nothing rather than a total spanning paths it cannot + // read. Callers treat the refusal as "no local signal". + check_scopes(&authed, || "resources:read".to_string())?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| ResourceTypeCount { resource_type: r.resource_type, count: r.count }) + .collect(), + )) +} + +/// A hub read, remembered with the hub it came from: `hub_base_url` is a live instance +/// setting, so a cache ignoring it would keep serving the previous hub's answers. +struct HubCached { + hub_base_url: String, + fetched_at: std::time::Instant, + value: T, +} + +/// The index changes only when a resource type is published to the hub; picks move slowly +/// and only reorder a list. Both are read on every drawer open, hence caching at all. +const HUB_RT_INDEX_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60); +const HUB_RT_PICKS_TTL: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// How long a *failed* index read is remembered. Short, and deliberately not the hour a +/// success is good for: this read is on the path of every picker open, so an unreachable +/// hub must not cost an outbound timeout each time, while one blip must not silence pick +/// reporting for an hour. +const HUB_RT_INDEX_FAILURE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +static HUB_RT_INDEX: LazyLock< + std::sync::RwLock>>>>, +> = LazyLock::new(|| std::sync::RwLock::new(None)); +static HUB_RT_PICKS: LazyLock>>>> = + LazyLock::new(|| std::sync::RwLock::new(None)); + +fn hub_cache_get( + cache: &std::sync::RwLock>>, + hub_base_url: &str, + ttl: std::time::Duration, +) -> Option { + let guard = cache.read().ok()?; + let entry = guard.as_ref()?; + (entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl) + .then(|| entry.value.clone()) +} + +fn hub_cache_put(cache: &std::sync::RwLock>>, hub_base_url: &str, value: T) { + if let Ok(mut guard) = cache.write() { + *guard = Some(HubCached { + hub_base_url: hub_base_url.to_string(), + fetched_at: std::time::Instant::now(), + value, + }); + } +} + +#[derive(Deserialize)] +struct HubResourceTypeEntry { + id: i64, + name: String, + /// Optional so a hub that does not send it costs only the mapping. Required, it would + /// fail the whole parse and take pick reporting — which needs just the id — with it. + #[serde(default)] + app: Option, +} + +#[derive(Clone)] +struct HubResourceType { + id: i64, + /// The integration the type belongs to. Usually the type's own name, but not always: + /// `discord_webhook` and `discord_bot_configuration` are both `discord`, and only the + /// hub knows that. Without it a workspace holding a `discord_webhook` resource looks + /// like one that has never touched Discord. + app: String, +} + +/// Reads the index cache, choosing the TTL by what is stored: a failure expires far sooner +/// than a success. `None` is a miss, `Some(None)` a remembered failure. +fn hub_index_cached(hub_base_url: &str) -> Option>> { + let guard = HUB_RT_INDEX.read().ok()?; + let entry = guard.as_ref()?; + if entry.hub_base_url != hub_base_url { + return None; + } + let ttl = if entry.value.is_some() { + HUB_RT_INDEX_TTL + } else { + HUB_RT_INDEX_FAILURE_TTL + }; + (entry.fetched_at.elapsed() < ttl).then(|| entry.value.clone()) +} + +/// What the hub knows about every published resource type, keyed by the name Windmill +/// addresses it by. `None` when the hub cannot be reached or does not answer with a list. +async fn hub_resource_types( + db: &DB, + hub_base_url: &str, +) -> Option> { + if let Some(cached) = hub_index_cached(hub_base_url) { + return cached; + } + let index = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/list"), + false, + None, + Some(db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + // Only the id and the app are kept. That listing carries every type's schema — + // around a megabyte — and neither reporting a pick nor grouping types by + // integration needs it. + Some( + response + .json::>() + .await + .ok()? + .into_iter() + .map(|rt| { + let app = rt.app.unwrap_or_else(|| rt.name.clone()); + (rt.name, HubResourceType { id: rt.id, app }) + }) + .collect::>(), + ) + } + .await; + + hub_cache_put(&HUB_RT_INDEX, hub_base_url, index.clone()); + index +} + +#[derive(Serialize)] +struct PickHubResourceTypeResult { + success: bool, +} + +/// Tells the hub a resource type was taken into a workspace, which is the counter its +/// `/resource_types/picked` ranking reads. +/// +/// Never fails the caller: a hub predating the route, an unreachable one, and a type that +/// is local-only all mean the same thing — not counted — and the request that reaches here +/// has already saved the user's resource. +/// +/// POST, and scoped as a write, because it changes state on the hub under the instance's +/// own credentials. The sibling `/type/*` routes are metadata reads that a `resources:run` +/// app-embed token may make, and both the method and this check keep such a token — which +/// is untrusted app JavaScript — from driving hub counters through us. +async fn pick_hub_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Path((_w_id, name)): Path<(String, String)>, +) -> JsonResult { + check_scopes(&authed, || "resources:write".to_string())?; + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let success = async { + let id = hub_resource_types(&db, &hub_base_url).await?.get(&name)?.id; + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/{id}/pick"), + false, + None, + Some(&db), + ) + .await + .ok()?; + Some(response.status().is_success()) + } + .await + .unwrap_or(false); + + if !success { + tracing::debug!("hub did not record a pick for resource type {name}"); + } + Ok(Json(PickHubResourceTypeResult { success })) +} + +#[derive(Deserialize, Clone)] +struct HubResourceTypePicks { + name: String, + /// The hub counts picks in a bigint, which its driver serialises as a string. + #[serde(deserialize_with = "windmill_common::more_serde::maybe_number")] + picks: i64, +} + +#[derive(Deserialize)] +struct HubPickedResourceTypes { + resource_types: Vec, +} + +#[cfg(test)] +mod hub_picks_tests { + use super::*; + + /// Two properties of the index cache that a later edit could quietly drop: it is keyed on + /// the hub it came from, and a failure is forgotten long before a success is. + #[test] + fn the_index_cache_is_keyed_on_the_hub_and_forgets_failures_sooner() { + let index = || { + Some(HashMap::from([( + "slack".to_string(), + HubResourceType { id: 1, app: "slack".to_string() }, + )])) + }; + + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", index()); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_some())); + // Switching hubs must miss rather than serve the previous hub's mapping. + assert!(hub_index_cached("https://other.example").is_none()); + + // A remembered failure reads as a hit (so the hub is not re-attempted) carrying + // nothing, and only until the shorter of the two TTLs. + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", None); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_none())); + assert!(HUB_RT_INDEX_FAILURE_TTL < HUB_RT_INDEX_TTL); + + if let Ok(mut guard) = HUB_RT_INDEX.write() { + *guard = None; + } + } + + /// The hub counts picks in a bigint, which postgres.js serialises as a string. Typing + /// the field as a plain i64 fails the whole response, and the ranking silently empties. + #[test] + fn picks_decode_from_a_string_or_a_number() { + let parsed: HubPickedResourceTypes = serde_json::from_str( + r#"{"resource_types":[{"name":"slack","picks":"42"},{"name":"github","picks":7}]}"#, + ) + .unwrap(); + assert_eq!( + parsed + .resource_types + .iter() + .map(|rt| (rt.name.as_str(), rt.picks)) + .collect::>(), + vec![("slack", 42), ("github", 7)] + ); + } +} + +/// One hub resource type as the pickers need it. +#[derive(Serialize)] +struct HubResourceTypeInfo { + name: String, + /// The integration it belongs to, so a caller can total a workspace's resources per + /// integration rather than per type. + app: String, + picks: i64, +} + +/// What the hub knows about its resource types: which integration each belongs to, and how +/// often each has been picked. +/// +/// Empty rather than an error when the hub answers neither read, so the pickers treat an +/// older or private hub as "no hub signal" and fall back to what the workspace itself uses. +/// The two reads degrade independently: a hub that lists types but has no `picked` route +/// still supplies the type-to-integration mapping, which is what decides whether a +/// workspace's resources are recognised as belonging to an integration at all. +async fn list_hub_resource_type_info( + Extension(db): Extension, +) -> JsonResult> { + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let picks = match hub_cache_get(&HUB_RT_PICKS, &hub_base_url, HUB_RT_PICKS_TTL) { + Some(picks) => picks, + None => { + let fetched = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/picked"), + false, + Some(vec![("limit", "200".to_string())]), + Some(&db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + Some( + response + .json::() + .await + .ok()? + .resource_types, + ) + } + .await + .unwrap_or_default(); + hub_cache_put(&HUB_RT_PICKS, &hub_base_url, fetched.clone()); + fetched + } + }; + + let mut picks_by_name: HashMap = + picks.into_iter().map(|rt| (rt.name, rt.picks)).collect(); + let index = hub_resource_types(&db, &hub_base_url) + .await + .unwrap_or_default(); + + let mut info: Vec = index + .into_iter() + .map(|(name, rt)| HubResourceTypeInfo { + picks: picks_by_name.remove(&name).unwrap_or(0), + name, + app: rt.app, + }) + .collect(); + // What the index did not account for is a type the picks read knows and the listing does + // not — which is what a hub answering only the second read looks like. Its own name is + // the same guess a caller makes for any type the mapping misses. + info.extend( + picks_by_name + .into_iter() + .map(|(name, picks)| HubResourceTypeInfo { app: name.clone(), name, picks }), + ); + + Ok(Json(info)) +} + async fn get_resource_type( authed: ApiAuthed, Extension(user_db): Extension, @@ -3065,8 +3599,10 @@ fn git_url_userinfo(url: &str) -> Option<&str> { git_url_userinfo_range(url).map(|r| &url[r]) } -/// Validates a git URL to prevent option injection, SSRF, and local file read. -async fn validate_git_url(url: &str) -> Result<()> { +/// Validates a git URL to prevent option injection, SSRF, and local file read. The +/// syntax and scheme checks apply to every caller; the private-host refusal only +/// where [`private_git_host_allowed`] refuses `caller`. +async fn validate_git_url(url: &str, caller: GitRemoteCaller) -> Result<()> { let url = url.trim(); if url.is_empty() { return Err(Error::BadRequest("Git URL cannot be empty".to_string())); @@ -3120,25 +3656,26 @@ async fn validate_git_url(url: &str) -> Result<()> { let host = extract_host_from_git_url(url) .ok_or_else(|| Error::BadRequest("Could not parse hostname from git URL".to_string()))?; - // CI/dev escape hatch: integration tests run their git remote (a Gitea - // container) on localhost, which the network-target checks below reject. - // Scheme and option-injection validation above still applies. - if std::env::var("ALLOW_LOCAL_GIT_REMOTES").is_ok_and(|v| v == "true" || v == "1") { + // Scheme and option-injection validation above applies to every caller. + if private_git_host_allowed(caller) { return Ok(()); } + let hint = private_git_host_hint(caller) + .map(|h| format!(" {h}")) + .unwrap_or_default(); if host == "localhost" || host.ends_with(".local") || host == "[::1]" { - return Err(Error::BadRequest( - "Git URLs targeting localhost or local network are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting localhost or local network are not allowed.{hint}" + ))); } // Check literal IP addresses if let Ok(ip) = host.parse::() { if is_private_or_reserved_ip(&ip) { - return Err(Error::BadRequest( - "Git URLs targeting private or reserved IP addresses are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting private or reserved IP addresses are not allowed.{hint}" + ))); } } else { // Hostname — resolve via DNS and reject if any address is private. Fail @@ -3159,9 +3696,9 @@ async fn validate_git_url(url: &str) -> Result<()> { } for addr in addrs { if is_private_or_reserved_ip(&addr.ip()) { - return Err(Error::BadRequest( - "Git URL hostname resolves to a private or reserved IP address".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URL hostname resolves to a private or reserved IP address.{hint}" + ))); } } } @@ -3263,8 +3800,24 @@ async fn get_git_commit_hash( .map_err(|e| { Error::BadRequest(format!("Invalid git repository resource format: {}", e)) })?; + let caller = if authed.is_admin { + GitRemoteCaller::AdminOrSystem + } else { + GitRemoteCaller::NonAdmin + }; git_resource.url = - resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false).await?; + resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false, caller) + .await?; + // A credential is stored under the repository it was issued for, so a + // resource repointed elsewhere finds none. Which credential can be attached + // is bounded by that; who may use it is bounded here, on the same terms as + // the installation credential above. + let plain_url = git_resource.url.clone(); + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(&db, &w_id, git_resource.url).await?; + if git_resource.url != plain_url { + require_admin(authed.is_admin, &authed.username)?; + } let identities: Vec = query .git_ssh_identity @@ -3284,7 +3837,7 @@ async fn get_git_commit_hash( let (git_ssh_cmd, filenames) = get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; - let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd, caller).await; delete_paths(&filenames).await; @@ -3388,12 +3941,17 @@ async fn get_git_ssh_cmd( const GIT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// `git` command for a remote probe, with HTTP redirects disabled. `validate_git_url` -/// only vets the host in the URL; git's default (`http.followRedirects=initial`) -/// would let a validated public remote 302 the probe onto a private or link-local -/// address that no check ever sees. Build every probe through this. +/// checks the host in the URL, never one a redirect names; git's default +/// (`http.followRedirects=initial`) would let a public remote 302 the probe of a +/// caller refused private hosts onto one. Build every probe through this. +/// +/// The transports are pinned too: an SCP-shaped remote-helper string such as +/// `ext::@host:path` passes the URL check for a caller allowed private +/// hosts, and only git's own config would stop it from running the command. fn git_probe_command() -> Command { let mut git_cmd = Command::new("git"); git_cmd.args(["-c", "http.followRedirects=false"]); + git_cmd.env("GIT_ALLOW_PROTOCOL", "http:https:ssh:git"); git_cmd } @@ -3453,8 +4011,8 @@ fn dot_git_url(url: &str) -> Option { } /// Run a remote probe, retrying against [`dot_git_url`] if the remote answered the -/// URL as given with a redirect. Extending the path keeps the retry on the host -/// `validate_git_url` already cleared, which is exactly what following the redirect +/// URL as given with a redirect. Extending the path keeps the retry on the host of +/// the URL `validate_git_url` checked, which is exactly what following the redirect /// would not guarantee. `build` must produce the probe for the URL it is handed. /// /// A retry that also fails reports the *original* failure, so the caller's message @@ -3594,6 +4152,7 @@ async fn resolve_azure_devops_url( w_id: &str, url: &str, allow_cache: bool, + caller: GitRemoteCaller, ) -> Result { // Trim first: the http(s) gates the callers apply trim too, so a stored URL with // leading whitespace must not reach the scheme check here as a non-http one. @@ -3606,7 +4165,7 @@ async fn resolve_azure_devops_url( // cost a live credential (nor cache one), and whoever can edit the URL would // otherwise drive a token mint per poll tick. let probe_url = url.replace(placeholder, "windmill"); - validate_git_url(&probe_url).await?; + validate_git_url(&probe_url, caller).await?; // The background poller reads the referenced resource under the system identity, // which bypasses RLS. Confining the destination is what keeps that from becoming an @@ -3806,9 +4365,10 @@ fn git_sync_system_dba(db: &DB) -> DbWithOptAuthed<'static, ApiAuthed> { async fn get_repo_latest_commit_hash( git_resource: &GitRepositoryResource, git_ssh_command: Option, + caller: GitRemoteCaller, ) -> Result { // Validate URL and branch to prevent option injection and SSRF attacks - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, caller).await?; let ref_spec = git_resource .branch @@ -3941,19 +4501,30 @@ pub async fn get_git_repo_head_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } + git_resource.url = resolve_azure_devops_url( + &git_sync_system_dba(db), + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; + // A repo whose credential Windmill holds carries none in its URL, so the + // poller has to attach it here or every probe would be unauthenticated. git_resource.url = - resolve_azure_devops_url(&git_sync_system_dba(db), w_id, &git_resource.url, true).await?; + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; if let Some(branch) = git_resource.branch.as_deref().filter(|s| !s.is_empty()) { let branch = branch.to_string(); - let sha = get_repo_latest_commit_hash(&git_resource, None).await?; + let sha = get_repo_latest_commit_hash(&git_resource, None, GitRemoteCaller::AdminOrSystem) + .await?; return Ok(Some((branch, sha))); } // No explicit branch: resolve the remote's default-branch NAME along with // its head in one call. Fork sync needs the concrete name to scope // `wm-fork//*`, so a bare "HEAD" ref would silently disable it. - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; let output = run_git_probe_for_url(&git_resource.url, "ls-remote --symref HEAD", |url| { let mut git_cmd = git_probe_command(); git_cmd.args(["ls-remote", "--symref", url, "HEAD"]); @@ -4048,8 +4619,20 @@ pub async fn get_git_repo_fork_heads_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } - git_resource.url = resolve_azure_devops_url(&dba, w_id, &git_resource.url, true).await?; - validate_git_url(&git_resource.url).await?; + git_resource.url = resolve_azure_devops_url( + &dba, + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; + // Same reason as the head probe above: a repository whose credential Windmill + // holds carries none in its URL, and listing the fork branches is the half of + // polling that would otherwise go out unauthenticated. + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; validate_git_ref(base_branch)?; for r in extra_refs { @@ -4435,46 +5018,52 @@ mod tests { )); } + // A caller let through to private hosts must still hit the scheme check. #[tokio::test] async fn test_validate_git_url_blocks_file_scheme() { - let result = validate_git_url("file:///etc/passwd").await; + let result = validate_git_url("file:///etc/passwd", GitRemoteCaller::AdminOrSystem).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("https://")); } #[tokio::test] async fn test_validate_git_url_blocks_private_ips() { - assert!(validate_git_url("http://127.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://169.254.169.254/latest/meta-data/") - .await - .is_err()); - assert!(validate_git_url("http://10.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://172.16.0.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("http://192.168.1.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("git://0.0.0.0/repo.git").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1/repo.git").await.is_err()); + assert!(v("http://169.254.169.254/latest/meta-data/").await.is_err()); + let err = v("http://10.0.0.1/repo.git").await.unwrap_err(); + assert!(err.to_string().contains("ALLOW_LOCAL_GIT_REMOTES"), "{err}"); + assert!(v("http://172.16.0.1/repo.git").await.is_err()); + assert!(v("http://192.168.1.1/repo.git").await.is_err()); + assert!(v("git://0.0.0.0/repo.git").await.is_err()); // IPv6 loopback, unique-local, and link-local literals - assert!(validate_git_url("git://[::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fd00::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fe80::1]/repo.git").await.is_err()); + assert!(v("git://[::1]/repo.git").await.is_err()); + assert!(v("git://[fd00::1]/repo.git").await.is_err()); + assert!(v("git://[fe80::1]/repo.git").await.is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_lets_admins_reach_private_hosts() { + assert!( + validate_git_url("http://10.0.0.1/repo.git", GitRemoteCaller::AdminOrSystem) + .await + .is_ok() + ); } #[tokio::test] async fn test_validate_git_url_blocks_localhost() { - assert!(validate_git_url("http://localhost/repo.git").await.is_err()); - assert!(validate_git_url("http://myhost.local/repo.git") - .await - .is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://localhost/repo.git").await.is_err()); + assert!(v("http://myhost.local/repo.git").await.is_err()); } #[tokio::test] async fn test_validate_git_url_blocks_local_paths() { - assert!(validate_git_url("/etc/passwd").await.is_err()); - assert!(validate_git_url("../relative/path").await.is_err()); - assert!(validate_git_url("./local/repo").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("/etc/passwd").await.is_err()); + assert!(v("../relative/path").await.is_err()); + assert!(v("./local/repo").await.is_err()); } /// Minimal loopback HTTP server: replies to every request with `response` and @@ -4604,7 +5193,11 @@ mod tests { async fn test_validate_git_url_fails_closed_on_unresolvable_host() { // `.invalid` never resolves (RFC 6761). The private-IP check is only // meaningful if a failed lookup rejects instead of falling through. - let result = validate_git_url("https://this-host-does-not-exist.invalid/repo.git").await; + let result = validate_git_url( + "https://this-host-does-not-exist.invalid/repo.git", + GitRemoteCaller::NonAdmin, + ) + .await; assert!( result.is_err(), "an unresolvable host was allowed — does this resolver synthesize records for NXDOMAIN?" @@ -4615,21 +5208,33 @@ mod tests { #[tokio::test] async fn test_validate_git_url_allows_valid_urls() { // Needs DNS: validation fails closed on a host it cannot resolve. - assert!(validate_git_url("https://github.com/user/repo.git") + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("https://github.com/user/repo.git").await.is_ok()); + assert!(v("git@github.com:user/repo.git").await.is_ok()); + assert!(v("ssh://git@github.com/user/repo.git").await.is_ok()); + } + + #[tokio::test] + async fn test_git_probe_refuses_remote_helpers() { + // A caller allowed private hosts skips the DNS step that would reject this + // SCP-shaped string, so the transport pin is what keeps git from running it. + let output = git_probe_command() + .args(["ls-remote", "testhelper::x@127.0.0.1:repo"]) + .output() .await - .is_ok()); - assert!(validate_git_url("git@github.com:user/repo.git") - .await - .is_ok()); - assert!(validate_git_url("ssh://git@github.com/user/repo.git") - .await - .is_ok()); + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("transport 'testhelper' not allowed"), + "{stderr}" + ); } #[tokio::test] async fn test_validate_git_url_blocks_option_injection() { - assert!(validate_git_url("-evil").await.is_err()); - assert!(validate_git_url("--upload-pack=evil").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("-evil").await.is_err()); + assert!(v("--upload-pack=evil").await.is_err()); } #[test] @@ -4725,24 +5330,21 @@ mod tests { // GHSA-p5cj-8cfh-mjv6: a loopback authority must stay blocked, and the // fragment/query `@public-host` bypasses of #8600 must be rejected so the // host git dials can never diverge from the validated host. - assert!(validate_git_url("http://127.0.0.1:40173/repo.git") - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - // A legitimate public repo URL still validates. + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1:40173/repo.git").await.is_err()); assert!( - validate_git_url("https://github.com/windmill-labs/windmill.git") + v("http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git") .await - .is_ok() + .is_err() ); + assert!( + v("http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git") + .await + .is_err() + ); + // A legitimate public repo URL still validates. + assert!(v("https://github.com/windmill-labs/windmill.git") + .await + .is_ok()); } } diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 3766a6050d..773616a807 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -158,6 +158,7 @@ pub struct RunJob { pub payload: JobPayload, pub args: serde_json::Map, pub scheduled_for_o: Option>, + pub username: String, pub email: String, pub job_id: Option, pub workspace_id: String, @@ -169,6 +170,7 @@ impl From for RunJob { payload, args: Default::default(), scheduled_for_o: None, + username: "test-user".to_string(), email: "test@windmill.dev".to_string(), job_id: None, workspace_id: "test-workspace".to_string(), @@ -190,7 +192,11 @@ impl RunJob { self } - pub fn email(mut self, email: impl Into) -> Self { + /// Run as this workspace member. Both halves together, because the job's identity is the + /// principal: an address paired with another member's username is re-resolved at push to + /// the address that username holds. + pub fn as_user(mut self, username: impl Into, email: impl Into) -> Self { + self.username = username.into(); self.email = email.into(); self } @@ -206,7 +212,7 @@ impl RunJob { } pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args, scheduled_for_o, email, job_id, workspace_id } = self; + let RunJob { payload, args, scheduled_for_o, username, email, job_id, workspace_id } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -219,9 +225,9 @@ impl RunJob { &workspace_id, payload, windmill_queue::PushArgs::from(&hm_args), - /* user */ "test-user", + /* user */ &username, /* email */ &email, - /* permissioned_as */ "u/test-user".to_string(), + /* permissioned_as */ format!("u/{username}"), /* token_prefix */ None, /* audit_end_user */ None, scheduled_for_o, diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 68986bb6f9..01ddfd4a81 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -9,7 +9,7 @@ use sqlx::PgConnection; use std::collections::HashSet; use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::global_settings::HTTP_ROUTE_WORKSPACED_ROUTE; +use windmill_common::global_settings::{validate_allowed_origins, HTTP_ROUTE_WORKSPACED_ROUTE}; use windmill_common::{ db::UserDB, error::{Error, Result}, @@ -189,6 +189,7 @@ pub async fn insert_new_trigger_into_db( authentication_resource_path, wrap_body, raw_string, + allowed_origins, script_path, summary, description, @@ -207,7 +208,7 @@ pub async fn insert_new_trigger_into_db( retry ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, now(), $21, $22, $23, $24 ) "#, w_id, @@ -218,6 +219,7 @@ pub async fn insert_new_trigger_into_db( trigger.config.authentication_resource_path, trigger.config.wrap_body.unwrap_or(false), trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.base.script_path, trigger.config.summary, trigger.config.description, @@ -444,6 +446,7 @@ impl TriggerCrud for HttpTrigger { "workspaced_route", "wrap_body", "raw_string", + "allowed_origins", ]; fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject { @@ -474,6 +477,8 @@ impl TriggerCrud for HttpTrigger { validate_authentication_method(new.authentication_method, new.raw_string)?; + validate_allowed_origins(new.allowed_origins.as_deref().unwrap_or_default())?; + Ok(()) } @@ -492,6 +497,8 @@ impl TriggerCrud for HttpTrigger { validate_authentication_method(edit.authentication_method, edit.raw_string)?; + validate_allowed_origins(edit.allowed_origins.as_deref().unwrap_or_default())?; + Ok(()) } @@ -554,33 +561,35 @@ impl TriggerCrud for HttpTrigger { workspaced_route = $3, wrap_body = $4, raw_string = $5, - authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - mode = $10, - http_method = $11, - static_asset_config = $12, - edited_by = $13, - permissioned_as = $14, - request_type = $15, - authentication_method = $16, - summary = $17, - description = $18, + allowed_origins = $6, + authentication_resource_path = $7, + script_path = $8, + path = $9, + is_flow = $10, + mode = $11, + http_method = $12, + static_asset_config = $13, + edited_by = $14, + permissioned_as = $15, + request_type = $16, + authentication_method = $17, + summary = $18, + description = $19, edited_at = now(), - is_static_website = $19, - error_handler_path = $20, - error_handler_args = $21, - retry = $22 + is_static_website = $20, + error_handler_path = $21, + error_handler_args = $22, + retry = $23 WHERE - workspace_id = $23 AND - path = $24 + workspace_id = $24 AND + path = $25 "#, route_path, &route_path_key, Some(effective_workspaced), trigger.config.wrap_body.unwrap_or(false), trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, @@ -613,30 +622,32 @@ impl TriggerCrud for HttpTrigger { SET wrap_body = $1, raw_string = $2, - authentication_resource_path = $3, - script_path = $4, - path = $5, - is_flow = $6, - mode = $7, - http_method = $8, - static_asset_config = $9, - edited_by = $10, - permissioned_as = $11, - request_type = $12, - authentication_method = $13, - summary = $14, - description = $15, + allowed_origins = $3, + authentication_resource_path = $4, + script_path = $5, + path = $6, + is_flow = $7, + mode = $8, + http_method = $9, + static_asset_config = $10, + edited_by = $11, + permissioned_as = $12, + request_type = $13, + authentication_method = $14, + summary = $15, + description = $16, edited_at = now(), - is_static_website = $16, - error_handler_path = $17, - error_handler_args = $18, - retry = $19 + is_static_website = $17, + error_handler_path = $18, + error_handler_args = $19, + retry = $20 WHERE - workspace_id = $20 AND - path = $21 + workspace_id = $21 AND + path = $22 "#, trigger.config.wrap_body.unwrap_or(false), trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs index b78276c28d..40bab102a5 100644 --- a/backend/windmill-trigger-http/src/lib.rs +++ b/backend/windmill-trigger-http/src/lib.rs @@ -8,7 +8,7 @@ use tokio::sync::{RwLock, RwLockReadGuard}; use windmill_common::{ error::{Error, Result}, flows::Retry, - global_settings::HTTP_ROUTE_WORKSPACED_ROUTE, + global_settings::{allows_any_origin, HTTP_ROUTE_WORKSPACED_ROUTE}, utils::ExpiringCacheEntry, worker::CLOUD_HOSTED, DB, @@ -51,6 +51,7 @@ pub struct TriggerRoute { pub workspaced_route: bool, pub wrap_body: bool, pub raw_string: bool, + pub allowed_origins: Option>, pub error_handler_path: Option, pub error_handler_args: Option>>, pub retry: Option>, @@ -127,6 +128,7 @@ pub struct HttpConfig { pub workspaced_route: bool, pub wrap_body: bool, pub raw_string: bool, + pub allowed_origins: Option>, } #[derive(Debug, Clone, Serialize)] @@ -144,6 +146,7 @@ pub struct HttpConfigRequest { pub workspaced_route: Option, pub wrap_body: Option, pub raw_string: Option, + pub allowed_origins: Option>, } #[derive(Deserialize)] @@ -162,6 +165,7 @@ struct HttpConfigRequestHelper { workspaced_route: Option, wrap_body: Option, raw_string: Option, + allowed_origins: Option>, } impl<'de> Deserialize<'de> for HttpConfigRequest { @@ -197,6 +201,7 @@ impl<'de> Deserialize<'de> for HttpConfigRequest { workspaced_route: helper.workspaced_route, wrap_body: helper.wrap_body, raw_string: helper.raw_string, + allowed_origins: helper.allowed_origins, }) } } @@ -216,6 +221,50 @@ pub struct RouteExists { pub workspaced_route: Option, } +/// The allowlist that governs a route: its own when it has one, otherwise the +/// instance-wide default. `None` means nothing is configured at either level, so +/// the route keeps the historical permissive behaviour. +/// +/// A list containing `*` is treated as no restriction, which is how a route opts +/// out of a stricter instance default. +pub fn effective_allowed_origins<'a>( + route_allowed_origins: Option<&'a [String]>, + instance_default: &'a [String], +) -> Option<&'a [String]> { + // An empty list is not a configuration. It reads exactly as never having set + // one, so such a route still inherits the instance default rather than + // skipping it, which is what would make `[]` more permissive than `NULL`. + match route_allowed_origins.filter(|list| !list.is_empty()) { + // `*` is the opt-out, including out of a stricter instance default. + Some(list) if allows_any_origin(list) => None, + Some(list) => Some(list), + None => (!instance_default.is_empty() && !allows_any_origin(instance_default)) + .then_some(instance_default), + } +} + +/// Resolve the `Access-Control-Allow-Origin` value for a request, or `None` to +/// omit the header so the browser blocks the read. +/// +/// The request's `Origin` is echoed back only on a match against the allowlist. +/// Reflecting it unchecked is the classic way this feature turns into no +/// restriction at all. +/// +/// The comparison ignores ASCII case because a browser lowercases the scheme and +/// host it sends, so a configured `https://App.Example.com` would otherwise name +/// a real origin and still match nothing. +pub fn match_origin( + allowed_origins: &[String], + origin: Option<&http::HeaderValue>, +) -> Option { + let origin = origin?; + let origin_str = origin.to_str().ok()?; + allowed_origins + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(origin_str)) + .then(|| origin.clone()) +} + pub fn validate_authentication_method( authentication_method: AuthenticationMethod, raw_string: Option, @@ -276,6 +325,7 @@ pub async fn refresh_routers( static_asset_config AS "static_asset_config: _", wrap_body, raw_string, + allowed_origins, workspaced_route, is_static_website, error_handler_path, @@ -384,6 +434,10 @@ pub struct HttpTrigger; #[cfg(test)] mod tests { use super::*; + // Not used by the lib itself, only exercised here. + use windmill_common::global_settings::{ + validate_allowed_origins, MAX_ALLOWED_ORIGINS, MAX_ALLOWED_ORIGIN_LEN, + }; #[test] fn test_request_type_backward_compatibility() { @@ -578,6 +632,168 @@ mod tests { assert!(validate_authentication_method(AuthenticationMethod::Signature, None).is_ok()); } + // --- CORS allowed origins --- + + fn origin(value: &str) -> http::HeaderValue { + http::HeaderValue::from_str(value).unwrap() + } + + #[test] + fn test_match_origin_exact_match_echoes_request_origin() { + let allowed = vec!["https://a.com".to_string(), "https://b.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://b.com"))), + Some(origin("https://b.com")) + ); + } + + #[test] + fn test_match_origin_ignores_case() { + let allowed = vec!["https://App.Example.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://app.example.com"))), + Some(origin("https://app.example.com")) + ); + } + + #[test] + fn test_match_origin_no_match_omits_header() { + let allowed = vec!["https://a.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://evil.com"))), + None + ); + // A prefix of an allowed origin must not match: https://a.com.evil.com + // is a different site entirely. + assert_eq!( + match_origin(&allowed, Some(&origin("https://a.com.evil.com"))), + None + ); + } + + #[test] + fn test_wildcard_entry_means_unrestricted() { + // `*` is handled before matching: it means "no restriction", which is + // how a route opts out of a stricter instance default. + assert!(allows_any_origin(&["*".to_string()])); + assert!(allows_any_origin(&[ + "https://a.com".to_string(), + "*".to_string() + ])); + assert!(!allows_any_origin(&["https://a.com".to_string()])); + assert_eq!( + effective_allowed_origins(Some(&["*".to_string()]), &[]), + None + ); + } + + #[test] + fn test_effective_allowed_origins_prefers_the_route() { + let route = ["https://a.com".to_string()]; + let default = ["https://default.com".to_string()]; + assert_eq!( + effective_allowed_origins(Some(&route), &default), + Some(&route[..]) + ); + // No route list: the instance default applies. + assert_eq!( + effective_allowed_origins(None, &default), + Some(&default[..]) + ); + // No route list and no instance default: nothing is restricted, so the + // historical permissive behaviour is kept. + assert_eq!(effective_allowed_origins(None, &[]), None); + // A route opting out with `*` escapes a stricter instance default. + assert_eq!( + effective_allowed_origins(Some(&["*".to_string()]), &default), + None + ); + // An empty route list is not a configuration: it resolves exactly as + // `NULL` does, so it inherits the instance default rather than skipping + // it and becoming more permissive than an unset one. + assert_eq!( + effective_allowed_origins(Some(&[]), &default), + Some(&default[..]) + ); + assert_eq!(effective_allowed_origins(Some(&[]), &[]), None); + } + + #[test] + fn test_match_origin_missing_origin_header_omits_header() { + let allowed = vec!["https://a.com".to_string()]; + assert_eq!(match_origin(&allowed, None), None); + } + + #[test] + fn test_validate_allowed_origins_accepts_anything_comparable() { + // A shape that cannot match simply matches nothing, so it is the + // editor's job to warn and not this one's to refuse. What is refused is + // narrower: `null`, values that are not header-comparable, entries that + // cannot round-trip the editor's comma-separated field, and lists past + // the size a request can afford to scan. + let allowed = vec![ + "https://app.example.com".to_string(), + "http://localhost:3000".to_string(), + "http://[::1]:8080".to_string(), + "chrome-extension://mhjfbmdgcfjbbpaeojofohoefgiehjai".to_string(), + // Never matches, but that is the caller's problem, not an error. + "https://app.example.com/".to_string(), + "https://app.example.com:99999".to_string(), + "not-an-origin".to_string(), + "*".to_string(), + ]; + assert!(validate_allowed_origins(&allowed).is_ok()); + assert!(validate_allowed_origins(&[]).is_ok()); + } + + #[test] + fn test_parse_allowed_origins_setting_rejects_empty_array_entries() { + use windmill_common::global_settings::parse_allowed_origins_setting; + // A trailing separator in the string form is a typing artifact and is + // dropped; an empty array entry is something the caller wrote, so it + // must reach validation rather than be filtered away into an empty + // (and therefore unrestricted) default. + assert!(parse_allowed_origins_setting(Some(&serde_json::json!("https://a.com,"))).is_ok()); + assert!(parse_allowed_origins_setting(Some(&serde_json::json!([""]))).is_err()); + assert!( + parse_allowed_origins_setting(Some(&serde_json::json!(["https://a.com", ""]))).is_err() + ); + } + + #[test] + fn test_validate_allowed_origins_bounds_the_list() { + // An allowlist is scanned on every request to a restricted route, the + // unauthenticated preflight included, so its size is a cost anyone can + // trigger. + let too_many = vec!["https://a.com".to_string(); MAX_ALLOWED_ORIGINS + 1]; + assert!(validate_allowed_origins(&too_many).is_err()); + assert!(validate_allowed_origins(&too_many[..MAX_ALLOWED_ORIGINS]).is_ok()); + let too_long = format!("https://{}.com", "a".repeat(MAX_ALLOWED_ORIGIN_LEN)); + assert!(validate_allowed_origins(&[too_long]).is_err()); + } + + #[test] + fn test_validate_allowed_origins_rejects_null_and_uncomparable() { + for invalid in [ + // Every sandboxed iframe sends `Origin: null`, so allowing it would + // grant access to any page that can open one. + "null", + "NULL", // Cannot be the string an Origin header is compared against. + "https://a b.com", + "https://app.example.com ", + "https://exämple.com", + // The editor edits the list as one comma-separated field, so an + // entry carrying a comma would come back as two and widen the list. + "https://a.com,https://b.com", + "", + ] { + assert!( + validate_allowed_origins(&[invalid.to_string()]).is_err(), + "expected {invalid} to be rejected" + ); + } + } + // --- Route path regex --- #[test] diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 8ef7c1ee39..953f3a571b 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -18,8 +18,9 @@ use windmill_common::{ error::{Error, JsonResult, Result}, trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource}, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows, - overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, + WithDraftQuery, }, utils::{paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, @@ -990,6 +991,18 @@ async fn delete_trigger( .await?; if !deleted { + drop(tx); + if delete_draft_only_for_path( + &db, + &workspace_id, + T::user_draft_item_kind(), + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only trigger '{}' deleted", path)); + } return Err(Error::NotFound(format!( "Trigger not found at path: {}", path diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index 463a365b06..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -14,17 +14,20 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, where `` is the name the - /// workspace configures it under. + /// A warehouse relation, `dbt:////`, where + /// `` is the name the workspace configures it under. /// - /// The SCHEME names the producer — dbt is the only thing that creates one — - /// while the PATH stays the physical relation, because that is what two - /// projects agree on: a mart one builds is a `source` the next reads, and - /// their dbt `unique_id`s differ (`model.a.orders` vs - /// `source.b.analytics.orders`) where the relation does not - /// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that - /// reader — the shared node is lineage, not a cascade edge. + /// The SCHEME names the namespace dbt made rather than an exclusive + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, + /// because that is what two producers agree on: a mart one builds is a + /// `source` the next reads, and their dbt `unique_id`s differ + /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does + /// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the + /// readers of what it built — that shared node is lineage, not a cascade + /// edge — while a declared write does (decision 25). Dbt, } diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 02feeb421a..cb1c53a579 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -538,6 +538,20 @@ pub struct Suspend { pub hide_cancel: Option, #[serde(skip_serializing_if = "false_or_empty")] pub continue_on_disapprove_timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skin: Option, +} + +/// How an approval request is presented, on the approval page and in Slack/Teams messages. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum ApprovalSkin { + Minimal, + /// A skin this server does not know renders as the detailed one rather than failing to + /// deserialize the whole flow, so a flow authored against a newer version still runs. + #[default] + #[serde(other)] + Detailed, } fn false_or_empty(v: &Option) -> bool { @@ -1365,6 +1379,23 @@ mod tests { assert_eq!(val.modules.len(), 1); } + #[test] + fn suspend_skin_unknown_value_falls_back_to_detailed() { + let skin_of = |skin: &str| { + let val: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": {"type": "identity"}, + "suspend": {"required_events": 1, "skin": skin} + }] + })) + .unwrap(); + val.modules[0].suspend.as_ref().unwrap().skin + }; + assert_eq!(skin_of("minimal"), Some(ApprovalSkin::Minimal)); + assert_eq!(skin_of("not_a_skin_yet"), Some(ApprovalSkin::Detailed)); + } + #[test] fn agent_tool_keeps_description_through_locking() { // #10244: the dependency job rebuilds each tool from its locked FlowModule; the diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 5e1639e242..9e29731505 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -16,7 +16,7 @@ use crate::{ runnable_settings::{ConcurrencySettings, DebouncingSettings}, }; -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] pub struct ScriptModule { pub content: String, pub language: ScriptLang, @@ -616,6 +616,7 @@ impl Hash for NewScript { self.priority.hash(state); self.timeout.hash(state); self.delete_after_use.hash(state); + self.delete_after_secs.hash(state); self.restart_unless_cancelled.hash(state); self.deployment_message.hash(state); self.visible_to_runner_only.hash(state); diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs index a9003a3712..e2e4ab866b 100644 --- a/backend/windmill-worker-volumes/src/lib.rs +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -179,11 +179,24 @@ pub fn interpolate_volume_name( result } +/// PHP's opening tag, which is case-insensitive and may be followed by code. +fn is_php_open_tag(trimmed_line: &str) -> bool { + trimmed_line + .get(..5) + .is_some_and(|p| p.eq_ignore_ascii_case(" volume: ` lines, +/// stopping at the first line that is neither blank nor a comment. A PHP script +/// opens with ` Vec { let mut volumes = Vec::new(); for line in content.lines() { let trimmed = line.trim(); - if trimmed.is_empty() { + if trimmed.is_empty() || is_php_open_tag(trimmed) { continue; } if !trimmed.starts_with(comment_prefix) { @@ -230,6 +243,17 @@ mod tests { ); } + #[test] + fn parse_php_volume_after_open_tag() { + let content = + " Result, Error> { - let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)? + let main_arg_signature = parse_sig_of_lang(content, Some(&language), None).await? .ok_or_else(|| Error::BadConfig(format!( "Cannot parse signature for language {:?}. The language parser may not be enabled in this build.", language diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index d948fee088..cd17c57f59 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -621,7 +621,7 @@ pub async fn handle_ai_agent_job( (schema, input_transforms, derived_description) } FlowModuleValue::RawScript { content, language, input_transforms, .. } => { - let schema = Some(parse_raw_script_schema(&content, &language)?); + let schema = Some(parse_raw_script_schema(&content, &language).await?); (schema, input_transforms, None) } FlowModuleValue::AIAgent { input_transforms, .. } => { @@ -1099,7 +1099,7 @@ pub async fn run_agent( // For non-Anthropic providers, response_format is handled by the query builder } - let user_wants_streaming = args.streaming.unwrap_or(false); + let user_wants_streaming = streaming_requested(args.streaming); *has_stream = user_wants_streaming && is_text_output; let mut final_events_str = String::new(); @@ -1701,6 +1701,17 @@ pub async fn run_agent( })) } +/// Whether the step asked for its answer as it is generated. Absence means on, matching the +/// schema's own default: a step that never wrote the key never had an opinion, and an answer +/// arriving as it is written is what people expect. Only an explicit `false` holds it back. +/// +/// The chat surfaces decide whether to open a stream from their own reading of the same config, +/// and a surface that opens one for an answer sent in a single piece re-runs the flow when the +/// connection times out. So this default is half of a contract, not a local preference. +fn streaming_requested(streaming: Option) -> bool { + streaming.unwrap_or(true) +} + #[cfg(test)] mod tests { use super::*; @@ -1713,6 +1724,13 @@ mod tests { } } + #[test] + fn an_unwritten_streaming_field_streams() { + assert!(streaming_requested(None)); + assert!(streaming_requested(Some(true))); + assert!(!streaming_requested(Some(false))); + } + /// Over 64 characters OpenAI rejects the key outright, which costs a wasted round /// trip per run and silently leaves that step with no prompt caching at all. #[test] diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index b3324f87ba..b577097c9d 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -408,16 +408,6 @@ pub fn create_empty_dir(path: &PathBuf) -> std::io::Result<()> { } } -/// Signals a detached `spawn_blocking` task that the future awaiting it is -/// gone, so it can stop instead of running to completion in the background. -struct AbortOnDrop(std::sync::Arc); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.store(true, std::sync::atomic::Ordering::Relaxed); - } -} - /// Lay down the tree of an app-backed repository, which git can't clone /// because its URL carries no credential. /// @@ -490,7 +480,7 @@ async fn fetch_repo_archive( // stopping it, so the flag is what a cancelled job uses to reach the // extraction loop. The guard sets it when this future is dropped. let aborted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _abort_on_drop = AbortOnDrop(aborted.clone()); + let _abort_on_drop = crate::common::AbortOnDrop(aborted.clone()); let unpack_archive = download_archive.clone(); tokio::task::spawn_blocking(move || { unpack_repo_archive(&unpack_archive, &download_target, &aborted) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 77221edf22..a034b0d184 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; use serde_json::value::RawValue; use uuid::Uuid; -use windmill_parser_ts::remove_pinned_imports; +use windmill_parser_ts::{remove_pinned_import_specifiers, remove_pinned_imports}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo}; @@ -1146,6 +1146,81 @@ pub async fn generate_bun_bundle( Ok(()) } +/// [`generate_bun_bundle`], built once more with the version pins dropped from the import +/// specifiers of `main.ts` if it fails. The lockfile pins those versions, but bun fails on a +/// pinned specifier except where it tolerates a failed import (in a `try`, under a `.catch`, in +/// dead code). Such a script builds as written and must keep that bundle, so only failures retry. +async fn generate_bun_bundle_unpinning_imports( + job_dir: &str, + w_id: &str, + job_id: &Uuid, + worker_name: &str, + db: Option<&Connection>, + timeout: Option, + mem_peak: &mut i32, + canceled_by: &mut Option, + common_bun_proc_envs: &HashMap, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, +) -> Result<()> { + let built = generate_bun_bundle( + job_dir, + w_id, + job_id, + worker_name, + db, + timeout, + mem_peak, + canceled_by, + common_bun_proc_envs, + occupancy_metrics, + ) + .await; + // Without a job, a failed build comes back as an `ExecutionErr`; with one, that variant is a + // cancellation or timeout, which must not be retried. + let build_failed = match &built { + Err(error::Error::ExitStatus(..)) => true, + Err(_) => db.is_none(), + Ok(()) => false, + }; + if !build_failed { + return built; + } + let Some(unpinned) = read_file_content(&format!("{job_dir}/main.ts")) + .await + .ok() + .and_then(|main| { + remove_pinned_import_specifiers(&main) + .ok() + .filter(|u| *u != main) + }) + else { + return built; + }; + write_file(job_dir, "main.ts", &unpinned)?; + if let Some(db) = db { + append_logs( + job_id, + w_id, + "\nbundling again with the imports' versions taken from the lockfile\n", + db, + ) + .await; + } + generate_bun_bundle( + job_dir, + w_id, + job_id, + worker_name, + db, + timeout, + mem_peak, + canceled_by, + common_bun_proc_envs, + occupancy_metrics, + ) + .await +} + struct PulledCodebase { is_esm: bool, } @@ -1305,7 +1380,7 @@ pub async fn prebundle_bun_script( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; - generate_bun_bundle( + generate_bun_bundle_unpinning_imports( job_dir, w_id, job_id, @@ -1784,8 +1859,19 @@ pub async fn handle_bun_job( if modules.as_ref().is_some_and(|m| !m.is_empty()) { let bundle_path = std::path::Path::new(job_dir).join("out").join("main.js"); if bundle_path.exists() { + // The lock-generation build kept every `pkg@version` specifier, and bun resolves + // a pinned specifier outside node_modules, loading a second copy of the package. + // The bundle holds the user's code too, so only the specifiers are rewritten, and + // a bundle the parser rejects still runs as built, pins and all. let bundled = std::fs::read_to_string(&bundle_path)?; - write_file(job_dir, "main.ts", &bundled)?; + let unpinned = remove_pinned_import_specifiers(&bundled).unwrap_or_else(|e| { + tracing::warn!( + job_id = %job.id, + "could not unpin the modules bundle, running it as built: {e:#}" + ); + bundled + }); + write_file(job_dir, "main.ts", &unpinned)?; } } "\n\n--- BUN CODE EXECUTION ---\n".to_string() @@ -1899,8 +1985,12 @@ pub async fn handle_bun_job( // Kept comment-free — this string is written out per job. // `_takePendingStepFailure` / `_takePendingSuspend` hand back what the body - // caught and swallowed; honour them instead of reporting a `complete` (see - // `_pendingStepFailure` in client.ts). Optional: npm clients may predate them. + // caught and swallowed; honour them instead of reporting a bare `complete` + // (see client.ts). Optional: npm clients may predate them. + // `_warnUnobservedTaskFailures` reports what the body never awaited, and so + // belongs only on the paths that end the round for good. A round that + // dispatches, sleeps, checkpoints or waits for approval replays later and + // re-registers the same failures from the checkpoint — keep those quiet. let wrapper_content = if is_wac_v2 { format!( r#" @@ -1958,6 +2048,7 @@ async function run() {{ if (trailing.length > 0) {{ return {{ type: "dispatch", mode: trailing.length > 1 ? "parallel" : "sequential", steps: trailing }}; }} + ctx._warnUnobservedTaskFailures?.(); return {{ type: "complete", result: result ?? null }}; }} catch (e) {{ setWorkflowCtx(null); @@ -1970,13 +2061,14 @@ async function run() {{ return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null, started_at: dispatch.started_at, duration_ms: dispatch.duration_ms }}; }} if (dispatch.mode === "approval") {{ - return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled }}; + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled, skin: dispatch.skin, description: dispatch.description }}; }} if (dispatch.mode === "sleep") {{ return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; }} return {{ type: "dispatch", mode: dispatch.mode ?? "sequential", steps: dispatch.steps ?? [] }}; }} + ctx._warnUnobservedTaskFailures?.(); const failed = ctx._takePendingStepFailure?.(); if (failed) {{ throw failed.error; @@ -2185,7 +2277,7 @@ try {{ if !codebase.is_some() && !has_bundle_cache { if build_cache { - generate_bun_bundle( + generate_bun_bundle_unpinning_imports( job_dir, &job.workspace_id, &job.id, @@ -2572,7 +2664,8 @@ try {{ // WAC v2 post-execution: parse output and handle dispatch/suspend if is_wac_v2 { - return handle_wac_v2_output(result, job, conn, modules, new_args.as_ref()).await; + return handle_wac_v2_output(result, job, conn, canceled_by, modules, new_args.as_ref()) + .await; } Ok(result) @@ -2602,11 +2695,13 @@ pub async fn handle_wac_v2_output( result: Box, job: &MiniPulledJob, conn: &Connection, + canceled_by: &mut Option, modules: &Option>, preprocessed_args: Option<&HashMap>>, ) -> error::Result> { use crate::wac_executor::{ - load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, WacOutput, + load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, + wac_cancelled_mid_segment, WacOutput, WacPark, }; use serde_json::Value; use windmill_common::get_latest_flow_version_info_for_path; @@ -2819,6 +2914,7 @@ pub async fn handle_wac_v2_output( // Step 1: Save checkpoint, suspend parent, and seed child checkpoints // in a single transaction — all BEFORE children become visible. + let segment_ms; { let mut tx = db.begin().await?; @@ -2871,24 +2967,24 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent before children become visible. - // Keep running = true so the normal pull query ignores it. - // The suspended pull query picks it up when suspend reaches 0 - // (it checks: suspend_until IS NOT NULL AND suspend <= 0). - let suspend_count = num_steps as i32; - sqlx::query!( - "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", - job.id, - suspend_count, + // Suspend parent before children become visible, so a child that + // completes immediately finds a parked parent to decrement. + match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + num_steps as i32, + 14.0 * 24.0 * 3600.0, ) - .execute(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Failed to suspend WAC parent job {}: {e}", - job.id - )) - })?; + .await? + { + WacPark::Parked(ms) => segment_ms = ms, + // Returning here drops `tx`, unwriting the checkpoint and the timeline + // entries, so no child is ever pushed against a parent that never parked. + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + } tx.commit().await?; } @@ -3167,10 +3263,17 @@ pub async fn handle_wac_v2_output( .execute(db) .await; - // Unsuspend parent so the error propagates instead of a 14-day hang + // Unsuspend parent so the error propagates instead of a 14-day hang. + // Unlike the other suspend exits this one completes the job for real, so + // it needs its segment start back — the in-memory copy is what the pull + // stamped, before the suspend cleared the column. let _ = sqlx::query!( - "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + "UPDATE v2_job_queue + SET suspend = 0, suspend_until = NULL, + started_at = coalesce(started_at, $2, now()) + WHERE id = $1", job.id, + job.started_at, ) .execute(db) .await; @@ -3183,12 +3286,13 @@ pub async fn handle_wac_v2_output( "WAC v2 parent job suspended" ); + crate::wac_executor::end_wac_segment(conn, job, segment_ms); Err(error::Error::WacSuspended(format!( "WAC v2 job {} suspended waiting for {} child job(s)", job.id, num_steps ))) } - WacOutput::Approval { key, timeout, form, self_approval_disabled } => { + WacOutput::Approval { key, timeout, form, self_approval_disabled, skin, description } => { let db = match conn { Connection::Sql(db) => db, _ => { @@ -3304,15 +3408,19 @@ pub async fn handle_wac_v2_output( }; // Store approval form metadata for the approval page endpoint - let approval_meta = serde_json::json!({ + let mut approval_meta = serde_json::json!({ "key": key, "form": form, "timeout": timeout_secs as u32, "self_approval_disabled": sad, + "skin": skin.unwrap_or_default(), "resume": resume_url, "cancel": cancel_url, "approvalPage": approval_page_url, }); + if let Some(description) = description.filter(|d| !d.is_null()) { + approval_meta["description"] = description; + } sqlx::query( "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( COALESCE(workflow_as_code_status, '{}'::jsonb), @@ -3361,15 +3469,23 @@ pub async fn handle_wac_v2_output( } // Suspend parent with suspend=1 (waiting for 1 approval event) - sqlx::query!( - "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", - job.id, + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, timeout_secs, ) - .execute(&mut *tx) - .await?; + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; tx.commit().await?; + crate::wac_executor::end_wac_segment(conn, job, segment_ms); tracing::info!( job_id = %job.id, @@ -3453,18 +3569,25 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent — it will auto-resume when suspend_until passes. // Use suspend=1 (not 0) so the suspended pull query only picks it up // when `suspend_until <= now()`, not via `suspend <= 0`. - sqlx::query!( - "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", - job.id, + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, sleep_secs, ) - .execute(&mut *tx) - .await?; + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; tx.commit().await?; + crate::wac_executor::end_wac_segment(conn, job, segment_ms); tracing::info!( job_id = %job.id, @@ -3514,19 +3637,25 @@ pub async fn handle_wac_v2_output( // Reset running=false so the job is immediately eligible for pickup. // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — // the job should be re-run right away to continue past the cached step. - sqlx::query!( - "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", + // `prev` holds the pre-update row: RETURNING would see the cleared column. + let segment_ms = sqlx::query_scalar!( + "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) + UPDATE v2_job_queue q SET running = false, started_at = null + FROM prev WHERE q.id = $1 + RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", job.id, ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await .map_err(|e| { error::Error::internal_err(format!( "Failed to reset running state for inline checkpoint: {e}" )) - })?; + })? + .flatten(); tx.commit().await?; + crate::wac_executor::end_wac_segment(conn, job, segment_ms); Err(error::Error::WacSuspended(format!( "WAC v2 job {} inline checkpoint for step {}", diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index f6391eab07..0a5deb52d2 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -67,6 +67,20 @@ mount { #[cfg(not(debug_assertions))] pub const DEV_CONF_NSJAIL: &str = ""; +/// Tells a `spawn_blocking` task to stop when the future awaiting it goes away. +/// +/// Dropping a `JoinHandle` detaches the task rather than cancelling it, so a +/// cancelled or timed-out phase otherwise leaves the blocking pool working on an +/// answer nobody will read. Hold one of these beside the handle and have the +/// blocking loop check the flag. +pub(crate) struct AbortOnDrop(pub(crate) std::sync::Arc); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.store(true, std::sync::atomic::Ordering::Relaxed); + } +} + /// Turn a JSON value into the string a shell/CLI arg should receive: a JSON string /// becomes its inner value, anything else is re-serialized compactly. pub(crate) fn raw_to_string(x: &str) -> String { diff --git a/backend/windmill-worker/src/dbt_column_index.rs b/backend/windmill-worker/src/dbt_column_index.rs new file mode 100644 index 0000000000..64f148534b --- /dev/null +++ b/backend/windmill-worker/src/dbt_column_index.rs @@ -0,0 +1,602 @@ +//! Column-level lineage and real column schemas, from the engine's own static +//! analysis. +//! +//! `manifest.json` carries neither. What does is the parquet index an engine +//! writes under `dbt compile --static-analysis strict --write-index`: +//! `dbt.column_lineage.parquet` (column-to-column edges, each labelled `copy`, +//! `mod` or `scan`) and `dbt.node_columns.parquet` (every column of every node, +//! typed and ordered, rather than only the ones an author documented). +//! +//! Four properties shape everything here, all of them measured against the real +//! engines rather than assumed: +//! +//! - **Strict analysis rejects SQL the default accepts.** An unresolvable +//! identifier is an error under `strict` and compiles fine otherwise, so this +//! is a SEPARATE pass with its own `--target-path`, never a flag on the build, +//! and it is opt-in per project. +//! - **A failed pass still writes the index**, holding every edge of the models +//! that did analyze. So the artifact is read whatever the exit status. +//! - **The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 accepts +//! `--write-index`, declares the views over these two tables in its own +//! `views.sql`, and writes neither file; only Fusion does today. Nothing here +//! asks which engine it is beyond "has the flag" — a release that starts +//! writing them is picked up with no change. +//! - **An incremental model has two shapes, and one ingest holds one of them.** +//! `is_incremental()` is false when the target does not exist or the build +//! is `--full-refresh`, so the `{{ this }}` self-join — and any `ref()` inside +//! that branch — compiles only in the other case. What this stores is +//! therefore what the compile in front of it saw: at DEPLOY, before the first +//! build, that is the cold shape, and a project deployed again after its +//! tables exist stores the incremental one for the same source. Nothing here +//! can reconcile that; dbt has no mode that emits both. The flag is taken from +//! the build so a per-run ingest matches its own run, and the version's graph +//! is honest about the compile that produced it rather than about every run +//! that will follow. + +use std::collections::HashSet; +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::path::Path; +use std::time::Duration; + +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::record::{Field, Row}; +use uuid::Uuid; +use windmill_common::dbt_manifest::{ + is_direct, ColumnIndex, IndexedColumn, IngestedColumnEdge, MAX_COLUMN_EDGES, +}; +use windmill_common::error; +use windmill_common::worker::Connection; +use windmill_parser_yaml::dbt::DbtDescriptor; +use windmill_queue::append_logs; + +use crate::dbt_executor::{dbt_command, Invocation, PreparedProject}; +use crate::handle_child::JobCtx; + +/// Where the lineage pass writes, relative to the project directory. +/// +/// Its own tree, not the runtime's `wm_target`: a `dbt compile` writes +/// `manifest.json` and `run_results.json` like any other invocation, and after a +/// build those two are what the graph ingest and `dbt retry` read. +const CLL_ARTIFACTS_DIR: &str = "wm_target_cll"; + +const COLUMN_LINEAGE_PARQUET: &str = "dbt.column_lineage.parquet"; +const NODE_COLUMNS_PARQUET: &str = "dbt.node_columns.parquet"; + +/// Run the lineage pass and read what it produced. +/// +/// Two steps with deliberately different contracts, because conflating them is +/// what made a best-effort annotation able to fail the job it annotates: +/// +/// - [`compile_index`] runs a subprocess and owns the JOB's semantics. Only a +/// cancellation or the job's own deadline can `Err` out of it; a non-zero exit +/// and an over-long output are outcomes, not failures. +/// - [`read_index`] owns the ARTIFACT's semantics. Reading it never fails the +/// job on the artifact's account: an absent, unreadable or partial index is a +/// value, not an error. It runs UNDER the poller all the same, so the job can +/// still end the phase — a cancel, a completion or the phase timeout — which +/// is the job's semantics reaching in, not the artifact's reaching out. +/// +/// The phase budget wraps the compile alone, because it exists to leave the +/// BUILD its share of the clock and only the compile can spend that share +/// unboundedly. The decode's own end is the job's: the poller it runs under +/// stops it when the job stops. +pub(crate) async fn collect( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + // The dbt subcommand the job runs, which decides the effective + // `--full-refresh` — see `dbt_executor::full_refresh`. + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + kept: &HashSet<&str>, +) -> error::Result> { + if !descriptor.column_lineage { + return Ok(None); + } + if !p.engine.engine.writes_column_index() { + append_logs( + job_id, + w_id, + format!( + "\n`column_lineage` is set, but the {} engine has no `--write-index`: column \ + lineage needs an engine that does static analysis. The rest of the graph is \ + unaffected.\n", + p.engine.engine.as_str() + ), + conn, + ) + .await; + return Ok(None); + } + + let index_dir = p.project_dir.join(CLL_ARTIFACTS_DIR).join("index"); + let Some(compiled) = compile_index(p, descriptor, inv, command, ctx, job_id, w_id, conn).await? else { + return Ok(None); + }; + let coverage = Coverage::of(&compiled); + + // Decoded UNDER the poller, not followed by a check of its own. The decode is + // the one phase of this pass with no subprocess behind it, so nothing else + // heartbeats while it runs: left alone, a large index is a silent worker for + // as long as it takes, which the zombie sweep reads as a dead job and + // restarts. The poller pings throughout and ends this with an `Err` if the + // job is cancelled or completed meanwhile — the job's own semantics, which + // this module may always propagate. + let artifact = crate::handle_child::run_future_with_polling_update_job_poller( + *job_id, + ctx.timeout(), + conn, + ctx.mem_peak, + ctx.canceled_by, + async { Ok(read_index(&index_dir, kept).await) }, + ctx.worker_name, + w_id, + &mut Some(ctx.occupancy_metrics), + Box::pin(futures::stream::empty()), + ) + .await?; + + // What only the pass knows. The COUNTS are logged where the index is folded + // into the graph, since the graph decides how much of it is kept. + let note = match artifact { + Artifact::Read(index) => { + if let Some(note) = coverage.caveat() { + log(job_id, w_id, note, &compiled.stderr, conn).await; + } + return Ok(Some(index)); + } + // The truncated arms come first: a compile stopped part-way explains an + // absent or unreadable artifact, and blaming the engine's capability + // for it sends the reader to check the wrong thing entirely. + Artifact::Missing if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass printed more than this runtime reads and was \ + stopped before it wrote `{COLUMN_LINEAGE_PARQUET}`." + ), + Artifact::Unreadable(why) if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass was stopped for printing more than this \ + runtime reads, and the `{COLUMN_LINEAGE_PARQUET}` it had written could not be read \ + ({why})." + ), + // Said apart from the one below, because it sends the reader somewhere + // else entirely: the engine did its job and this runtime could not read + // what it wrote. + Artifact::Unreadable(why) => format!( + "No column lineage: `{COLUMN_LINEAGE_PARQUET}` was written but could not be read \ + ({why}). The graph is unaffected." + ), + Artifact::Missing => format!( + "No column lineage: the analysis pass wrote no `{COLUMN_LINEAGE_PARQUET}`. Only an \ + engine that computes it does, and only for the warehouses it analyzes natively — \ + the flag alone is not the capability." + ), + }; + // The engine's own diagnostics come along. They are how a reader learns that + // this adapter turned static analysis off, which it reports as a warning on + // a SUCCESSFUL compile that nothing else would show. + log(job_id, w_id, ¬e, &compiled.stderr, conn).await; + Ok(None) +} + +async fn log(job_id: &Uuid, w_id: &str, note: &str, stderr: &str, conn: &Connection) { + append_logs( + job_id, + w_id, + format!("\n{note}\n{}", diagnostics(stderr)), + conn, + ) + .await; +} + +/// How completely the analysis compile covered the project. +/// +/// Every way the COMPILE can disappoint is a value here rather than an error. An +/// `Err` from `compile_index` is the JOB's — a cancellation or its deadline — +/// and must fail it; the pass giving up on its own terms is `Ok(None)` and has +/// already been logged. +enum Coverage { + /// Every model analyzed. + Whole, + /// `--static-analysis strict` rejected part of the project. Whatever it did + /// analyze is still in the index. + Partial, + /// The output ceiling killed the compile mid-run. Distinct from `Partial`: + /// nothing rejected the project, but the index is however far it had got, so + /// it is not `Whole` either. + Truncated, +} + +impl Coverage { + fn of(c: &crate::dbt_executor::Captured) -> Self { + match (c.truncated, c.success) { + (true, _) => Coverage::Truncated, + (false, true) => Coverage::Whole, + (false, false) => Coverage::Partial, + } + } + + /// What to tell the reader when an index WAS produced. `None` for a run that + /// covered everything, which needs no caveat. + fn caveat(&self) -> Option<&'static str> { + match self { + Coverage::Whole => None, + Coverage::Partial => Some( + "Column lineage: `--static-analysis strict` rejected part of the project, so \ + the lineage covers only the models it could analyze.", + ), + Coverage::Truncated => Some( + "Column lineage: the analysis pass printed more than this runtime reads and was \ + stopped, so the lineage covers only the models it had reached.", + ), + } + } +} + +/// Run `dbt compile --static-analysis strict --write-index`, under this phase's +/// share of the job's clock. +/// +/// `Ok(None)` is "the pass gave up and said so"; `Err` is the job's own +/// cancellation or deadline and must propagate. Nothing outlives this function: +/// the budget is a race around the child, and dropping that future kills it +/// through `run_captured`'s `kill_on_drop`. +async fn compile_index( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result> { + // A previous pass in the same job directory — a retry's second attempt — + // would otherwise be read back as this one's answer. + tokio::fs::remove_dir_all(p.project_dir.join(CLL_ARTIFACTS_DIR)) + .await + .ok(); + + // Static analysis can log in to read schemas, so it gets live credentials like + // every other dbt process. + if let Err(e) = p.refresh_profile(descriptor, job_id, w_id, conn).await { + append_logs( + job_id, + w_id, + format!("\nColumn lineage: skipped, {e}\n"), + conn, + ) + .await; + return Ok(None); + } + + let mut cmd = dbt_command( + p, + &[ + "compile", + "--static-analysis", + "strict", + "--write-index", + // Documented as what builds the CLL graph, and `--write-index` alone + // happens to imply it on the engine probed. Passed explicitly so the + // pass does not depend on which of the two is doing the work. + "--write-lineage", + "--target-path", + CLL_ARTIFACTS_DIR, + ], + ); + // The flag already wins over the env var dbt_command sets, but setting both + // means this pass cannot write into the runtime's artifacts even if that + // precedence ever changes — and what is in there after a build is the + // `run_results.json` a `dbt retry` resumes from. + cmd.env("DBT_TARGET_PATH", CLL_ARTIFACTS_DIR); + crate::dbt_executor::add_vars(&mut cmd, descriptor, inv)?; + // The BUILD's answer, not the descriptor's default: `is_incremental()` + // branches on it, so a model reading `{{ this }}` compiles its self-join — + // and any `ref()` inside that branch — only when this is absent. Guessing + // here stores lineage for SQL the run never executed. + if crate::dbt_executor::full_refresh(descriptor, inv, command)? { + cmd.arg("--full-refresh"); + } + // Captured rather than streamed: a strict-analysis failure is a wall of + // diagnostics about SQL the build itself accepts, and this pass decides + // nothing about whether that build runs. + // Read before the future below borrows `ctx` mutably. + let budget = phase_budget(ctx); + let run = crate::dbt_executor::run_captured( + cmd, + "dbt compile (column lineage)", + ctx, + job_id, + w_id, + conn, + CLL_MAX_OUTPUT_BYTES, + // The ceiling is this pass's, not the job's: a compile that prints more + // than it than we care to read has still analyzed the project, and the + // index it wrote is on disk either way. + crate::dbt_executor::Overflow::Truncate, + ); + let Some(budget) = budget else { + return Ok(Some(run.await?)); + }; + match tokio::time::timeout(budget, run).await { + Ok(r) => Ok(Some(r?)), + Err(_) => { + append_logs( + job_id, + w_id, + format!( + "\nNo column lineage: the analysis pass did not finish within {}s, half of \ + what was left of this job's time. The build below gets the rest.\n", + budget.as_secs() + ), + conn, + ) + .await; + Ok(None) + } + } +} + +/// stdout the pass may produce. It is a compile, so this is diagnostics rather +/// than data. +const CLL_MAX_OUTPUT_BYTES: usize = 1 << 20; + +/// The share of the job's remaining wall clock this pass may spend. +/// +/// A per-run refresh ingests BEFORE the build and shares the job's one deadline, +/// so an unbounded pass on a slow project would hand `dbt build` an expired +/// budget and fail the run it exists only to annotate. Half leaves the build at +/// least as long as the annotation was allowed to take. +/// +/// Spent as a race around the COMPILE rather than as a shortened deadline handed +/// to the runner: the runner reports its expiry as an `Err`, indistinguishable +/// from a cancellation or the job's own deadline, and those two MUST fail the +/// job. Expiring here is this budget and nothing else. The child dies with the +/// dropped future through `run_captured`'s `kill_on_drop`; the decode is outside +/// this race and answers to the poller instead. +fn phase_budget(ctx: &JobCtx<'_>) -> Option { + ctx.timeout() + .map(|left| Duration::from_secs((left.max(0) as u64 / 2).max(1))) +} + +/// The tail of what the engine said, bounded. The whole of it is every rendered +/// model on a large project, which is not what a job log is for. +const DIAGNOSTIC_LINES: usize = 40; + +fn diagnostics(out: &str) -> String { + let lines: Vec<&str> = out.lines().filter(|l| !l.trim().is_empty()).collect(); + let tail = &lines[lines.len().saturating_sub(DIAGNOSTIC_LINES)..]; + match tail.is_empty() { + true => String::new(), + false => format!("{}\n", tail.join("\n")), + } +} + +/// What came back from the artifact. Never an `Err`: nothing the file does or +/// fails to do is a reason to fail a job. `Unreadable` is separate from +/// `Missing` because the two send a reader looking in different places — one at +/// their engine and adapter, the other at a file that exists. +enum Artifact { + Read(ColumnIndex), + Missing, + Unreadable(String), +} + +/// Read both parquets, if the lineage one is there. +/// +/// The column schemas alone are not worth a graph: they arrive with the lineage +/// or not at all, and a node's declared columns already answer for the case +/// where the pass never ran. +async fn read_index(index_dir: &Path, kept: &HashSet<&str>) -> Artifact { + let lineage = index_dir.join(COLUMN_LINEAGE_PARQUET); + if !tokio::fs::try_exists(&lineage).await.unwrap_or(false) { + return Artifact::Missing; + } + let columns = index_dir.join(NODE_COLUMNS_PARQUET); + // Owned, because the decode moves to a blocking thread. The index describes + // the whole project while this graph describes one selection of it, so + // scoping HERE is what keeps the bound below from being spent on rows the + // graph would discard anyway. + let kept: HashSet = kept.iter().map(|s| (*s).to_string()).collect(); + // Dropping the handle of a blocking task does NOT stop it: the poller + // cancelling this phase would otherwise leave a thread decoding millions of + // rows for a job that is over. `abandoned` is set when this future is + // dropped, and the row loop reads it. + let abandoned = Arc::new(AtomicBool::new(false)); + let _stop = crate::common::AbortOnDrop(abandoned.clone()); + // Decompressing and decoding a parquet is CPU work on a file the engine just + // wrote, so it does not belong on the runtime's poll thread. + let read = tokio::task::spawn_blocking(move || { + read_index_blocking(&lineage, &columns, &kept, &abandoned) + }) + .await; + match read { + Ok(Ok(index)) => Artifact::Read(index), + Ok(Err(e)) => Artifact::Unreadable(e.to_string()), + Err(e) => Artifact::Unreadable(e.to_string()), + } +} + +fn read_index_blocking( + lineage: &Path, + columns: &Path, + kept: &HashSet, + abandoned: &AtomicBool, +) -> error::Result { + let mut out = ColumnIndex::default(); + // ONE pass, with the two kinds bucketed as they arrive. `copy` and `mod` say + // the value itself travelled, so they get the whole budget; `scan` — the + // column was read to produce the ROW, which reaches every output column of + // its model and is the bulk of a wide project's index — fills only what is + // left over at the end. Reading the file twice to get that ordering would + // double the decode of exactly the large index this bound exists for. + let mut scan: Vec = Vec::new(); + for_each_row(lineage, abandoned, |row| { + let lineage_kind = string(row, "lineage_kind"); + let parent_unique_id = string(row, "from_node_unique_id"); + let child_unique_id = string(row, "to_node_unique_id"); + let parent_column = string(row, "from_column_name"); + let child_column = string(row, "to_column_name"); + // A column of a node the analysis could not name is not an endpoint the + // graph can draw, and neither is one outside this graph's nodes. + if parent_column.is_empty() + || child_column.is_empty() + || !kept.contains(&parent_unique_id) + || !kept.contains(&child_unique_id) + { + return ControlFlow::Continue(()); + } + let edge = IngestedColumnEdge { + parent_unique_id, + parent_column, + child_unique_id, + child_column, + lineage_kind, + }; + // The bound covers BOTH buckets, so the pass never holds more than one + // budget's worth however the kinds are distributed. + let held = out.edges.len() + scan.len(); + if is_direct(&edge.lineage_kind) { + // A direct edge displaces a `scan` one: the budget is spent on + // value flow first. + if held >= MAX_COLUMN_EDGES { + scan.pop(); + } + out.edges.push(edge); + // The edge that FILLS the budget ends the read, not the next one to + // arrive: once the displacing kind is full nothing later in the file + // can be kept, and waiting for another direct edge to say so decodes + // a `scan`-only tail all the way to the backstop for nothing. + return match out.edges.len() >= MAX_COLUMN_EDGES { + true => ControlFlow::Break(()), + false => ControlFlow::Continue(()), + }; + } + if held < MAX_COLUMN_EDGES { + scan.push(edge); + } + // Not a stopping point even when full: a direct edge still to come takes + // a `scan` entry's place. + ControlFlow::Continue(()) + })?; + out.edges.append(&mut scan); + // Absent is normal — an engine can write the lineage table and not this one — + // and unreadable is not worth losing the lineage over. + let mut held = 0usize; + let _ = for_each_row(columns, abandoned, |row| { + let unique_id = string(row, "unique_id"); + let name = string(row, "column_name"); + if held >= MAX_INDEXED_COLUMNS { + return ControlFlow::Break(()); + } + if name.is_empty() || !kept.contains(&unique_id) { + return ControlFlow::Continue(()); + } + held += 1; + // The author's `data_type` where `schema.yml` gives one, since that is + // what the project calls the column; the analysis's own inference + // otherwise. + let column_type = match string(row, "declared_type") { + t if !t.is_empty() => t, + _ => string(row, "inferred_type"), + }; + out.columns + .entry(unique_id) + .or_default() + .push(IndexedColumn { + name, + column_type, + index: int(row, "column_index").unwrap_or(i64::MAX), + }); + ControlFlow::Continue(()) + }); + Ok(out) +} + +/// The most rows of `dbt.node_columns.parquet` one pass keeps. One per column of +/// the project, so the same bound as the edges is far more than any project +/// reaches; it exists for the same reason. +const MAX_INDEXED_COLUMNS: usize = MAX_COLUMN_EDGES; + +/// The most rows of an index one pass DECODES, whatever it keeps of them. +/// +/// A bound on work rather than on memory, and the two are separate because the +/// input this defends against is the one that cannot be collected: `scan` +/// lineage is emitted from every predicate and join column to every output +/// column, so a project shaped that way writes an index whose row count is +/// quadratic in its widest model. This pass runs outside the phase budget, on a +/// blocking thread, and nothing the file contains may fail a deploy or a run — +/// so the file it walks needs an end even when almost nothing in it is +/// retained. The abandonment flag ends it sooner when the job is over; this is +/// the bound for a job that is not. +const MAX_INDEX_ROWS: usize = 4_000_000; + +/// Decode a parquet a row at a time, handing each to `f` and never holding two. +/// +/// Collecting first would put a `Vec` — each row carrying its own copy of +/// every column NAME — in front of the caller's own bound, which is what would +/// take the worker process down on the index described above. +/// +/// `f` says when it has all it will take, and that is the ordinary end: this +/// runs outside the phase budget, so every row decoded past the point of being +/// able to keep one is wall clock the build below does not get. +fn for_each_row( + path: &Path, + abandoned: &AtomicBool, + mut f: impl FnMut(&Row) -> ControlFlow<()>, +) -> error::Result<()> { + let fail = |e: parquet::errors::ParquetError| { + error::Error::internal_err(format!("reading {}: {e}", path.display())) + }; + let file = std::fs::File::open(path) + .map_err(|e| error::Error::internal_err(format!("opening {}: {e}", path.display())))?; + let reader = SerializedFileReader::new(file).map_err(fail)?; + for (n, row) in reader.get_row_iter(None).map_err(fail)?.enumerate() { + // Nobody is waiting for this any more — the job was cancelled, completed + // or ran out of time while it decoded. + if abandoned.load(Ordering::Relaxed) { + break; + } + if n >= MAX_INDEX_ROWS { + tracing::warn!( + "dbt column index: {} holds more than {MAX_INDEX_ROWS} rows; the rest is dropped", + path.display() + ); + break; + } + if f(&row.map_err(fail)?).is_break() { + break; + } + } + Ok(()) +} + +/// By NAME, not by position: these tables are the engine's own schema and it +/// adds columns to them between releases. +fn field<'a>(row: &'a Row, name: &str) -> Option<&'a Field> { + row.get_column_iter() + .find(|(k, _)| k.as_str() == name) + .map(|(_, v)| v) +} + +fn string(row: &Row, name: &str) -> String { + match field(row, name) { + Some(Field::Str(s)) => s.clone(), + Some(Field::Bytes(b)) => String::from_utf8_lossy(b.data()).into_owned(), + _ => String::new(), + } +} + +fn int(row: &Row, name: &str) -> Option { + match field(row, name) { + Some(Field::Long(v)) => Some(*v), + Some(Field::Int(v)) => Some(*v as i64), + Some(Field::Short(v)) => Some(*v as i64), + Some(Field::UInt(v)) => Some(*v as i64), + Some(Field::ULong(v)) => i64::try_from(*v).ok(), + _ => None, + } +} diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 33ddd6c32e..8119897d6d 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -19,12 +19,13 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::client::AuthedClient; use windmill_common::error::{self, Error}; +use windmill_common::jobs::JobKind; use windmill_common::materialization::{ record_materialization, MaterializationStatus, RecordMaterializationRequest, }; use windmill_common::worker::{to_raw_value, write_file, Connection}; use windmill_parser_yaml::{ - parse_dbt_descriptor, DbtDescriptor, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, + parse_dbt_descriptor, DbtDescriptor, DbtEngine, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, DBT_COMMAND_LABEL, DBT_DEFAULT_WAREHOUSE, }; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -37,6 +38,9 @@ use crate::dbt_engine::{provision_engine, ProvisionedEngine, DBT_CACHE_DIR}; use crate::dbt_profiles::{ ensure_adapter_licensed, render_dbt_profile, render_profile, DbtAdapter, KnownAdapter, }; +use crate::dbt_state::{ + environment_label, prepare_deferral, write_state_dir, Deferral, StateManifest, STATE_DIR, +}; use crate::handle_child::{ get_mem_peak, handle_child, run_future_with_polling_update_job_poller, JobCtx, JobDeadline, }; @@ -121,6 +125,12 @@ pub struct DbtRunResult { /// the same project — cannot get them from the job. #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub invocation_args: std::collections::HashMap>, + /// The run whose stored state this one resolved its unbuilt `ref()`s + /// through, absent when it deferred to none. What a deferring run built + /// against is otherwise unrecoverable: the state is replaced by the next + /// successful run of that environment. + #[serde(skip_serializing_if = "Option::is_none")] + pub deferred_to: Option, } #[derive(Serialize, Debug, Default)] @@ -189,7 +199,13 @@ pub(crate) async fn handle_dbt_job( // result publishes, and both describe an invocation of this script, not one // executor's view of it. let raw_args = job.args.as_ref().map(|a| a.0.clone()).unwrap_or_default(); - let inv = Invocation { args: args.clone(), raw_args, envs: envs.clone(), strict: true }; + let inv = Invocation { + args: args.clone(), + raw_args, + envs: envs.clone(), + deferral: None, + strict: true, + }; // One wall clock for the whole job. A dbt job is a sequence of // subprocesses — provision, deps, parse, ls, build, then the // `after_all` tests — and each would otherwise resolve the job's full @@ -262,6 +278,16 @@ pub(crate) async fn handle_dbt_job( // applies — nothing is built, so there is no test phase, no materialization, // no retry state and no ownership to publish. if command == "parse" { + // Checked here rather than at the seam below, which cannot tell a parse + // from a run that simply left `defer` off: a parse never reaches the + // deferral at all, so it is the one caller for which "turn `defer` on" + // would be advice that leads nowhere. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + StateAccess::Never(&command), + !selection_is_overridden(&descriptor, &inv.args)?, + )?; return run_parse_only( &prepared, &descriptor, @@ -364,6 +390,84 @@ pub(crate) async fn handle_dbt_job( inv }; + // Read AFTER the retry restore, so a retry defers exactly as the run it + // resumes did: a retry's own arguments are the command block alone, and the + // relations its unbuilt `ref()`s resolve to must not depend on that. + let defer = arg_bool(&inv.args, "defer")?.unwrap_or(descriptor.defer); + // Before the state is fetched, not only at the seam where the selection + // reaches dbt: a selector that cannot work whatever the state says would + // otherwise be masked by the "nothing published yet" refusal, which sends the + // caller to publish a state that will not help. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + if defer { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(&descriptor, &inv.args)?, + )?; + // A `show` defers too, and every engine takes the flags on it: it COMPILES + // the model it previews, so a model whose upstream this environment built and + // this run did not is exactly the case a deferral exists for. + let inv = if defer { + // Refused before anything runs. `dbt retry` reads the run it resumes + // from `--state`, the flag a deferral needs, so an engine without + // `--defer-state` can be given one or the other: told to defer, it + // resumes the stored state's own (successful) results and rebuilds + // nothing, and left alone it rebuilds the failed nodes with every + // `ref()` resolving into the schema THIS run writes — which for the + // narrowed run a deferral exists to serve is not where those models go. + if command == "retry" && !prepared.engine.engine.has_defer_state_flag() { + return Err(Error::BadRequest(format!( + "`{}` cannot resume a run that deferred: `dbt retry` takes the run it resumes \ + from `--state`, which is also where a deferral reads its manifest, and this \ + engine has no `--defer-state` to tell the two apart. Run the script again \ + instead of resuming it, or move the project to dbt-core-1x", + prepared.engine.engine.as_str() + ))); + } + let deferral = prepare_deferral(&prepared, &job.workspace_id, job_dir, conn).await?; + // Only answerable once the state is loaded: `defer` is enough for a + // `state:` method, which reads the manifest every publication carries, + // but a `result:` one reads `run_results.json` — and a build recovered by + // node retry publishes without it, since the results it holds describe + // only the nodes the retry rebuilt. dbt-core then raises an INTERNAL + // error and the Rust engines match nothing and exit 0. + if !deferral.has_run_results + && selection_names( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + &["result"], + ) + { + return Err(Error::BadRequest(format!( + "a `result:` selector reads `run_results.json` out of the published state, and \ + the state for this environment ({}) carries only the manifest run {} \ + published: a build recovered by node retry stores none, its results describing \ + the retried nodes rather than the whole build. Run this script once without \ + `defer` and without overrides to publish a complete state, or drop the selector", + environment_label(&prepared), + deferral.published_by + ))); + } + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nDeferring unbuilt refs to the dbt state published by run {}; this run \ + publishes none of its own\n", + deferral.published_by + ), + conn, + ) + .await; + Invocation { deferral: Some(deferral), ..inv } + } else { + inv + }; + // Ingested BEFORE the build, from a `dbt parse` with this run's vars, so the // models shown are the ones about to be built. Rows are keyed by path, version // AND job so no two runs collide; the path-keyed `asset` usage belongs to one @@ -387,7 +491,7 @@ pub(crate) async fn handle_dbt_job( // For a retry the restored manifest already describes the invocation // being resumed, so only the ingest runs — with that invocation's // arguments, which the selection resolver needs to interpolate. - ingest_from_run(&prepared, &descriptor, &inv, &mut ctx, job, conn).await?; + ingest_from_run(&prepared, &descriptor, &inv, &command, &mut ctx, job, conn).await?; } // A read-only command prints rows to stdout, so it is captured rather than @@ -428,9 +532,28 @@ pub(crate) async fn handle_dbt_job( // previous attempt's `run_results.json` is still in the job directory. Never on // an agent worker, which cannot read `v2_job_queue` — the wait below would be // uninterruptible, so a cancelled job would hold its slot and then start dbt. + // And never where the engine cannot be told to defer on a `retry`: the + // rebuild would resolve this run's unbuilt refs into the schema it writes + // into, so the nodes it "recovered" would read from the wrong relations. + // Said out loud below rather than silently skipped. + let retry_would_lose_the_deferral = + inv.deferral.is_some() && !prepared.engine.engine.has_defer_state_flag(); let node_retry = descriptor .retry_failed_nodes - .filter(|_| matches!(conn, Connection::Sql(_))); + .filter(|_| matches!(conn, Connection::Sql(_))) + .filter(|_| !retry_would_lose_the_deferral); + if descriptor.retry_failed_nodes.is_some() && retry_would_lose_the_deferral { + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nSkipping the automatic node retry: `{}` cannot defer on a `dbt retry`\n", + prepared.engine.engine.as_str() + ), + conn, + ) + .await; + } let mut retries_left = node_retry.map(|p| p.attempts()).unwrap_or(0); if let Some(policy) = node_retry.filter(|_| run.is_err()) { retry_failed_nodes( @@ -519,6 +642,56 @@ pub(crate) async fn handle_dbt_job( { tracing::warn!("dbt: could not save retry state for job {}: {e:#}", job.id); } + // What a later run defers to, published by the runs whose relations are the + // SCRIPT's — the same condition that decides whether a run's graph becomes + // what the script owns, and for the same reason: an invocation that scoped + // its own models has no standing to say where this project's relations live. + // Success is the other half, because a relation a deferral resolves to has + // to exist. A `retry` is excluded: its `run_results.json` names only the + // nodes it redid, so publishing it would leave the environment claiming a + // run of a handful of models. + // + // And never a run that DEFERRED, whatever narrowed it. A deferring run built + // some of the relations its manifest names and resolved the rest out of the + // state it read, so publishing that manifest would record relations nothing + // built — and a model renamed since would be recorded under a name only a + // full build creates, breaking every later deferral until one repairs it. + // `publishes_ownership` cannot see this on its own: it reads the caller's + // overrides, and a descriptor that already narrows `select` needs none. + if run.is_ok() + && command == "build" + && inv.deferral.is_none() + // A run of the DEPLOYED version, by kind. A preview carries a + // caller-supplied `script_hash` into `runnable_id` + // (`run_preview_script`), so the version guard alone would let anyone who + // may run a job publish arbitrary content as a deployed script's state. + && job.kind == JobKind::Script + && prepared.graph_refresh.publishes_ownership() + { + // Losing it costs the next deferral, not the run that just finished — + // but silently, so the one actionable case (an artifact too large for + // the database on an instance with no object storage) says so. + if let Err(e) = crate::dbt_state::publish( + &prepared, + &job.workspace_id, + &job.id, + job.runnable_id.map(|h| h.0), + // An attempt was spent, so `run_results.json` on disk is the one + // `dbt retry` left: the nodes it redid, not the build. + node_retry.is_some_and(|p| retries_left < p.attempts()), + conn, + ) + .await + { + append_logs( + &job.id, + &job.workspace_id, + format!("\nCould not publish this run as the environment's dbt state: {e}\n"), + conn, + ) + .await; + } + } let reconciled = reconcile_materializations(&prepared, &results, job, conn, client).await; terminalize_running_relations(job, &reconciled, conn).await; @@ -653,12 +826,26 @@ pub(crate) async fn dbt_dep( None => GraphPublisher::Unversioned, }; let superseded = if let Some(warehouse) = prepared.warehouse.as_deref() { - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, prepared.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + &prepared, + &descriptor, + &inv, + // A deploy resolves the project by parsing it; nothing is built, so + // the pass takes the descriptor's own answer. + "parse", + &mut ctx, + job_id, + w_id, + &conn, + ) + .await?; let published = persist_ingest( db, w_id, @@ -682,6 +869,7 @@ pub(crate) async fn dbt_dep( &conn, ) .await; + warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await; } !published } else { @@ -866,6 +1054,9 @@ impl GraphRefresh { if selection_is_overridden(descriptor, args)? { self.per_run_models = true; } + if full_refresh_is_overridden(descriptor, args)? { + self.per_run_models = true; + } Ok(()) } } @@ -894,6 +1085,17 @@ pub struct PreparedProject { /// The descriptor's `profile.target`, passed as `--target` so it applies to /// a project-owned `profiles.yml` as well as a rendered one. pub target: Option, + /// The target dbt actually runs, which is the above only when the descriptor + /// names one: otherwise it is the workspace warehouse's, or the project's own + /// `profiles.yml` default. Half of an environment's identity, since a + /// `target.name` macro decides where a model is built. + pub effective_target: Option, + /// Whether the profile templates where its relations go — a project-owned + /// `profiles.yml`, a `dbt_profile` resource's block, or `profile.schema`, + /// all of which reach dbt as written. Two renderings then share one + /// `relation_root` and an environment cannot be told apart, so such a + /// project neither publishes state nor defers to any. + pub templated_location: bool, /// The profile target's database. Nodes that override it qualify their /// `dbt://` schema segment so two databases cannot collapse onto one node. pub default_database: Option, @@ -919,19 +1121,76 @@ pub struct PreparedProject { /// Written nsjail profile for this job, when the worker sandboxes jobs. /// `None` means the phases run unsandboxed, exactly as before. pub sandbox_config: Option, - /// One-way digest of the rendered profile — the resolved connection, not - /// just the names it exposes. A resource repointed from one warehouse to - /// another that happens to use the same database and schema names is - /// invisible to `relation_root`, and a retry would then execute the saved - /// failures against a warehouse where the successful nodes do not exist. + /// One-way digest of the rendered profile, credentials masked: the resolved + /// connection, not just the names it exposes. A resource repointed from one + /// warehouse to another that happens to use the same database and schema + /// names is invisible to `relation_root`, and a retry would then execute the + /// saved failures against a warehouse where the successful nodes do not exist. pub profile_digest: String, + /// The job's client, which `refresh_profile` re-resolves the warehouse with. + client: AuthedClient, } impl PreparedProject { + /// Re-resolve the warehouse and rewrite `profiles.yml` just before a dbt + /// process that logs in. What preparation wrote can have expired by then: a + /// Snowflake OAuth token lasts ten minutes, and the build follows `dbt deps` + /// and a parse, the `after_all` tests follow the build, a node retry follows + /// its backoff. + pub(crate) async fn refresh_profile( + &self, + descriptor: &DbtDescriptor, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + ) -> error::Result<()> { + // A project-owned `profiles.yml` is rendered by dbt itself, from an + // environment resolved once. + if descriptor.profile.profiles_yml.is_some() { + return Ok(()); + } + let fresh = match write_profiles( + descriptor, + &self.project_dir, + &self.project_dir.to_string_lossy(), + &self.client, + &self.template_env(), + ) + .await + { + Ok(fresh) => fresh, + // The profile on disk is still whole: a credential that does not + // expire connects with it exactly as before. + Err(e) => { + append_logs( + job_id, + w_id, + format!( + "\nCould not re-resolve the warehouse, so this dbt process uses the \ + credentials resolved earlier in the job: {e}\n" + ), + conn, + ) + .await; + return Ok(()); + } + }; + // Credentials are masked out of the digest, so a mismatch is the warehouse + // itself moving mid-run, and this process would build somewhere else. + if fresh.digest != self.profile_digest { + return Err(Error::BadRequest(format!( + "the `{}` warehouse was repointed while this run was in progress; run the \ + script again", + self.warehouse.as_deref().unwrap_or(DBT_DEFAULT_WAREHOUSE) + ))); + } + Ok(()) + } + /// Where this run's relations live: the resolved schema and database. Drift /// here since the deploy means the stored graph names relations that no /// longer exist. - fn relation_root(&self) -> String { + pub(crate) fn relation_root(&self) -> String { format!( "{}|{}", self.default_schema.as_deref().unwrap_or(""), @@ -1063,8 +1322,8 @@ pub(crate) async fn prepare_project( .chain(invocation_env.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect(); - let (profiles_dir, warehouse, adapter, default_database, default_schema, profile_digest) = - write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let profile = write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let adapter = profile.adapter.clone(); // The lockfile's version, when it pinned one for this same engine — a // descriptor edited to another engine invalidates the pin. let pinned_version = locks @@ -1185,18 +1444,21 @@ pub(crate) async fn prepare_project( h.finish() }, sandbox_config, - profile_digest, + profile_digest: profile.digest, + client: client.clone(), project_dir, - profiles_dir, + profiles_dir: profile.dir, engine, graph_refresh, - warehouse, + warehouse: profile.warehouse, target: descriptor.profile.target.clone(), + effective_target: profile.target, + templated_location: profile.templated_location, descriptor_content: descriptor_content.to_string(), descriptor_env, - default_database, - default_schema, + default_database: profile.database, + default_schema: profile.schema, script_path: script_path.to_string(), env, }; @@ -1511,6 +1773,24 @@ async fn strip_git_remote(dir: &Path) -> std::io::Result<()> { tokio::fs::write(&config, out).await } +/// What resolving the run's connection settled, beyond the file itself. +struct ResolvedProfile { + dir: PathBuf, + /// The workspace warehouse's NAME, when this project belongs to one. + warehouse: Option, + adapter: DbtAdapter, + database: Option, + schema: Option, + /// The target dbt actually runs, which is not always the descriptor's: it + /// falls back to the workspace warehouse's, and to the project's own + /// `profiles.yml` default. Resolved because it is half of an environment's + /// identity and a `target.name` macro can move every relation. + target: Option, + /// Whether a project-owned `profiles.yml` templates where its relations go. + templated_location: bool, + digest: String, +} + /// Write `profiles.yml`, either rendered from a Windmill resource or taken from /// the project itself. Both paths are supported (decision 8): the workspace /// warehouse is the ergonomic one, the project's own file is what makes an @@ -1521,14 +1801,7 @@ async fn write_profiles( job_dir: &str, client: &AuthedClient, template_env: &HashMap, -) -> error::Result<( - PathBuf, - Option, - DbtAdapter, - Option, - Option, - String, -)> { +) -> error::Result { // The workspace's warehouse, always: a descriptor names one by NAME or takes // `main`, and cannot name a resource at all. The NAME is what asset identity // keys on, so every project on one warehouse shares its nodes while the @@ -1609,14 +1882,16 @@ async fn write_profiles( } None => None, }; - return Ok(( + return Ok(ResolvedProfile { dir, - identity, + warehouse: identity, adapter, - target.database, - target.schema, - profile_digest, - )); + database: target.database, + schema: target.schema, + target: Some(target.name), + templated_location: target.templated_location, + digest: profile_digest, + }); } use windmill_common::workspaces::DBT_PROFILE_RESOURCE_TYPE; @@ -1666,9 +1941,6 @@ async fn write_profiles( .or(workspace_target.as_deref()) .unwrap_or("default"); let dir = PathBuf::from(job_dir).join("dbt_profiles"); - tokio::fs::create_dir_all(&dir) - .await - .map_err(|e| Error::internal_err(format!("creating the profiles dir: {e}")))?; let rendered = if is_dbt_profile { let block = value.as_object().ok_or_else(|| { Error::BadRequest( @@ -1689,6 +1961,7 @@ async fn write_profiles( } else { render_profile( &adapter, + descriptor.engine(), &value, &profile_name, target, @@ -1697,6 +1970,10 @@ async fn write_profiles( &dir, )? }; + // After the render, so a render that fails leaves the previous profile whole. + fresh_dir(&dir) + .await + .map_err(|e| Error::internal_err(format!("creating the profiles dir: {e}")))?; write_file(dir.to_str().unwrap(), "profiles.yml", &rendered.yaml)?; if let Some(pem) = rendered.root_certificate_pem.as_deref() { write_file( @@ -1706,19 +1983,41 @@ async fn write_profiles( )?; } let profile_digest = profile_identity_digest( - &rendered.yaml, + &rendered.identity, &dir, rendered.root_certificate_pem.as_deref(), &client.token, ); - Ok(( + Ok(ResolvedProfile { dir, - Some(warehouse.to_string()), + warehouse: Some(warehouse.to_string()), adapter, - rendered.database, - rendered.schema, - profile_digest, - )) + // A `dbt_profile` resource is one block of the user's own + // `profiles.yml`, copied through unchanged, and `profile.schema` is + // written as given — so either can be a template dbt renders and this + // runtime does not, exactly as a project-owned file can. + templated_location: [rendered.database.as_deref(), rendered.schema.as_deref()] + .iter() + .any(|v| v.is_some_and(is_jinja)), + database: rendered.database, + schema: rendered.schema, + target: Some(target.to_string()), + digest: profile_digest, + }) +} + +/// An empty directory at `dir`, whatever was there. `refresh_profile` writes into +/// the job directory after project code has run in a jail that can write it, and a +/// symlink left at `dir` or inside it would carry the worker's write out of the +/// sandbox. An entry that is not a real directory is unlinked, never followed. +async fn fresh_dir(dir: &Path) -> std::io::Result<()> { + match tokio::fs::symlink_metadata(dir).await { + Ok(m) if m.is_dir() => tokio::fs::remove_dir_all(dir).await?, + Ok(_) => tokio::fs::remove_file(dir).await?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + tokio::fs::create_dir_all(dir).await } /// Where a workspace warehouse name points: its resource path and, if the @@ -1740,7 +2039,9 @@ async fn resolve_warehouse( .map_err(|e| Error::BadRequest(format!("resolving the dbt warehouse `{warehouse}`: {e}"))) } -/// Identifies the connection a rendered profile describes, for run identity. +/// Identifies the connection a rendered profile describes, for run identity, +/// from the rendering whose credentials are already masked +/// (`RenderedProfile::identity`). /// /// Two things in the rendered text belong to the ATTEMPT rather than the /// connection, and hashing either as-is makes a retry reject its own @@ -1859,13 +2160,45 @@ async fn adapter_from_profiles_yml( // identically to one on a workspace warehouse, which is what lets the two // meet on the same node when they are on the same relation. let (database_key, schema_key) = adapter.target_identity_keys(); - let read = |k: &str| { + let raw = |k: &str| { out.get(k) .and_then(|v| v.as_str()) - .map(|v| v.to_string()) - .filter(|v| !v.is_empty() && !v.contains("{{")) + .filter(|v| !v.is_empty()) }; - Ok(ProfileTarget { adapter, database: read(database_key), schema: read(schema_key) }) + let read = |k: &str| raw(k).filter(|v| !v.contains("{{")).map(|v| v.to_string()); + Ok(ProfileTarget { + adapter, + database: read(database_key), + schema: read(schema_key), + // A TEMPLATED location is one dbt renders and this runtime does not, so + // two renderings of this file resolve to one `relation_root` and would + // share one environment — `{{ }}` because `read` drops it and it reads + // as absent, `{% %}` because the raw block is kept and reads the same + // for every rendering. Distinguished from plainly absent, which is the + // adapter's default and does not move. + templated_location: [database_key, schema_key] + .iter() + .any(|k| raw(k).is_some_and(is_jinja)), + // The output actually chosen, which for a templated `target:` is the sole + // one rather than the template text no output answers to. + name: match ( + templated_target, + outputs.as_mapping().and_then(|m| m.keys().next()), + ) { + (true, Some(only)) => only.as_str().unwrap_or(target).to_string(), + _ => target.to_string(), + }, + }) +} + +/// Whether dbt would RENDER this value rather than take it literally. +/// +/// Both delimiters, because dbt renders a profile through Jinja: `{{ … }}` +/// substitutes and `{% … %}` branches, and a schema spelled +/// `{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}` moves +/// every relation exactly as an `env_var()` does. +fn is_jinja(v: &str) -> bool { + v.contains("{{") || v.contains("{%") } /// What a project-owned `profiles.yml` target says, for the two things Windmill @@ -1876,6 +2209,14 @@ struct ProfileTarget { adapter: DbtAdapter, database: Option, schema: Option, + /// The output this resolved to, by name. + name: String, + /// Whether its database or schema is a template rather than a literal. The + /// fields above cannot say: a `{{ }}` value is dropped and reads as absent, + /// a `{% %}` block is kept and reads the same for every rendering. So this + /// is what separates "the adapter's default, which does not move" from + /// "wherever this run's environment renders it to". + templated_location: bool, } lazy_static::lazy_static! { @@ -2006,7 +2347,11 @@ pub(crate) fn dbt_command(p: &PreparedProject, args: &[&str]) -> Command { .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("GIT_PATH", GIT_PATH.as_str()); + .env("GIT_PATH", GIT_PATH.as_str()) + // dbt reports anonymous usage to dbt Labs from every invocation unless told + // not to, and a project's `flags:` block cannot override the variable. Set + // before the project's environment so a descriptor can still opt back in. + .env("DBT_SEND_ANONYMOUS_USAGE_STATS", "false"); // Both environments belong to the child. Under a sandbox they reach it through // the jail profile instead: set here, they would reach the dynamic loader that // execs nsjail itself, so an `LD_PRELOAD` from the project would run as the @@ -2191,6 +2536,29 @@ async fn retry_failed_nodes( } } +/// The flags that point a deferring invocation at its state directory. +/// +/// `--state` is where a deferred `ref()` resolves through — except on a `retry`, +/// which reads the run it RESUMES from that same flag: handed the deferral's +/// directory, dbt resumes the successful run stored there and rebuilds nothing. +/// dbt-core 1.x has `--defer-state` for exactly this split; the Rust engines do +/// not, and a run that defers is refused a retry there rather than rebuilt with +/// its refs resolving into the schema it writes into (`handle_dbt_job`), which +/// is why the last arm never fires in practice. +/// +/// The directory is relative because dbt records the invocation's flags into +/// `run_results.json`: an absolute path would name the job directory of the run +/// being resumed, gone by the time anything reads it back. +fn defer_flags(command: &str, engine: DbtEngine) -> &'static [&'static str] { + match command { + // `--defer` itself is restored with the rest of the resumed + // invocation's arguments and cannot be set from here. + "retry" if engine.has_defer_state_flag() => &["--defer-state", STATE_DIR], + "retry" => &[], + _ => &["--defer", "--state", STATE_DIR], + } +} + #[allow(clippy::too_many_arguments)] async fn run_dbt( p: &PreparedProject, @@ -2202,6 +2570,8 @@ async fn run_dbt( ctx: &mut JobCtx<'_>, with_selection: bool, ) -> error::Result<()> { + p.refresh_profile(descriptor, &job.id, &job.workspace_id, conn) + .await?; let mut cmd = dbt_command(p, &[command]); // The console stays human-readable and goes straight to the job log; the // machine-readable copy goes to a file the progress reporter tails, so @@ -2212,6 +2582,10 @@ async fn run_dbt( .args(["--log-format-file", "json"]) .args(["--log-level-file", p.engine.engine.progress_log_level()]); + if inv.deferral.is_some() { + cmd.args(defer_flags(command, p.engine.engine)); + } + if with_selection && command != "retry" { add_selection(&mut cmd, descriptor, inv)?; } @@ -2232,8 +2606,7 @@ async fn run_dbt( if let Some(t) = descriptor.threads { cmd.args(["--threads", &t.to_string()]); } - let full_refresh = arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh); - if full_refresh && command != "test" { + if full_refresh(descriptor, inv, command)? { cmd.arg("--full-refresh"); } } @@ -2862,7 +3235,11 @@ async fn run_show( `@`) or wildcard (`*`), resolves to a set: run `build` with it instead" ))); } + p.refresh_profile(descriptor, job_id, w_id, conn).await?; let mut cmd = dbt_command(p, &["show"]); + if inv.deferral.is_some() { + cmd.args(defer_flags("show", p.engine.engine)); + } add_vars(&mut cmd, descriptor, inv)?; // Intersected with `resource_type:model`, because `show` is only read-only // for models: dbt dispatches a selected SEED through its seed runner and @@ -2882,7 +3259,8 @@ async fn run_show( conn, SHOW_MAX_OUTPUT_BYTES, ) - .await?; + .await? + .stdout; // dbt frames the rows as `{"node": …, "show": [ … ]}`, pretty-printed, with a // banner before and a deprecation summary after — so neither "the line starting // with `{`" nor "first `{` to the end" parses. A streaming deserializer stops at @@ -2956,6 +3334,7 @@ fn build_result( totals, nodes, invocation_args: inv.raw_args.clone(), + deferred_to: inv.deferral.as_ref().map(|d| d.published_by), } } @@ -3064,7 +3443,7 @@ async fn run_parse_only( // manifest and the selection while the warehouse only keys them — so a project // with no warehouse identity still reports what dbt found. The placeholder // reaches no row: the guard below returns before anything is written. - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, p.warehouse.as_deref().unwrap_or("unkeyed"), p.default_database.as_deref(), @@ -3084,6 +3463,20 @@ async fn run_parse_only( else { return Ok(to_raw_value(&result)); }; + // AFTER the guard: the pass is a second `dbt compile` and a parquet decode, + // and a parse that stores nothing has nowhere to put what it would produce. + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + "parse", + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; match conn { Connection::Sql(db) => match job.runnable_id.map(|h| h.0) { Some(script_hash) => { @@ -3143,11 +3536,70 @@ async fn run_parse_only( Ok(to_raw_value(&result)) } +/// Fold this project's column lineage into the graph about to be stored, when +/// the descriptor asked for it. +/// +/// One helper for all three ingests — deploy, editor parse, per-run refresh — +/// because a graph that carries column lineage in one provenance and not another +/// reads as the lineage having disappeared. +async fn attach_column_index( + ingested: &mut windmill_common::dbt_manifest::IngestedManifest, + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result<()> { + // The nodes this graph kept, so the pass reads only rows it could store: the + // index describes the whole project, this graph one selection of it. + let kept: std::collections::HashSet<&str> = ingested + .nodes + .iter() + .map(|n| n.unique_id.as_str()) + .collect(); + let index = + crate::dbt_column_index::collect( + p, descriptor, inv, command, ctx, job_id, w_id, conn, &kept, + ) + .await?; + drop(kept); + let Some(index) = index else { + return Ok(()); + }; + let found = index.edges.len(); + ingested.attach_column_index(index); + let kept = ingested.column_edges.len(); + let typed: usize = ingested + .nodes + .iter() + .filter(|n| n.column_schema.is_some()) + .count(); + // Counted here rather than at the pass: the index describes the whole + // project and this graph describes one selection of it, so `found` is what + // dbt produced and `kept` is what the graph can draw. + let dropped = match found.saturating_sub(kept) { + 0 => String::new(), + n => format!(" ({n} outside this graph or past the cap)"), + }; + append_logs( + job_id, + w_id, + format!("\nIngested {kept} column lineage edges{dropped} and typed {typed} nodes\n"), + conn, + ) + .await; + Ok(()) +} + /// Refresh the stored graph from the manifest this run produced. async fn ingest_from_run( p: &PreparedProject, descriptor: &DbtDescriptor, inv: &Invocation, + command: &str, ctx: &mut JobCtx<'_>, job: &MiniPulledJob, conn: &Connection, @@ -3164,12 +3616,24 @@ async fn ingest_from_run( // filter this run's manifest by a different node set than it built. let selected = resolve_selection(p, descriptor, inv, ctx, &job.id, &job.workspace_id, conn).await?; - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, p.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + command, + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; // Only a run whose models are its own snapshots per run. A static // descriptor at a moved profile re-ingests the VERSION's graph, since the // move outlives the run; one that neither drifted nor overrode anything @@ -3177,7 +3641,7 @@ async fn ingest_from_run( let snapshot_job = p.graph_refresh.snapshot_job(job.id); match conn { Connection::Sql(db) => { - persist_ingest( + let published = persist_ingest( db, &job.workspace_id, script_path, @@ -3190,6 +3654,13 @@ async fn ingest_from_run( p.graph_refresh.publishes_ownership(), ) .await?; + // Publishing ownership from a RUN makes this project the owner of + // those relations exactly as a deploy does, so it can be what leaves + // a subscription accepted while the relation had no producer with dbt + // as its only one. Same warning the deploy emits. + if published && p.graph_refresh.publishes_ownership() { + warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await; + } } // An agent worker reaches these tables only through the API. Publishing // is the whole of what it needs: a worker that can replace the graph @@ -3242,7 +3713,7 @@ enum GraphPublisher { /// Replace this script's graph, unless a newer version of it has been deployed. /// /// Write one ingest: the sidecar rows and the `asset` usages the manifest -/// implies. No subscriptions — a `dbt://` one could never fire. +/// implies. No subscriptions — a dbt project is not woken by the cascade. /// /// Returns whether this job was still the one entitled to the path-keyed half — /// false once a newer version has superseded it, or once the version is gone. @@ -3333,9 +3804,10 @@ async fn persist_ingest( &ingested.assets, ) .await?; - // A `dbt://` subscription can never fire, so none are derived from the - // manifest. The delete stays to clear what earlier versions wrote, which would - // otherwise keep drawing cascade arrows that wake nothing. + // A dbt project is not woken by the asset cascade (refused at deploy), so + // none are derived from the manifest either. The delete stays to clear what + // earlier versions wrote, which would otherwise keep drawing cascade arrows + // that wake nothing. sqlx::query!( "DELETE FROM script_trigger WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2 @@ -3349,6 +3821,56 @@ async fn persist_ingest( Ok(true) } +/// Log the `// on dbt://…` subscriptions this project's relations leave dormant. +/// +/// Subscribing to a relation dbt already owns is refused at the subscriber's +/// deploy, but one deployed while nothing produced that relation is accepted — +/// as it is for every other asset kind — and an ingest is what can afterwards +/// make dbt its only producer. A dbt run does not dispatch, so such an edge is +/// drawn on the canvas and never fires; a job's own log is where that ordering is +/// visible. +/// +/// Called from both points that publish ownership, the deploy and a run whose +/// static descriptor found its profile moved — either can be the one that claims +/// the relation. +async fn warn_dormant_subscribers( + db: &sqlx::Pool, + w_id: &str, + job_id: &Uuid, + ingested: &windmill_common::dbt_manifest::IngestedManifest, + conn: &Connection, +) { + use windmill_common::assets::AssetUsageAccessType; + let relations: Vec = ingested + .assets + .iter() + .filter(|a| { + matches!( + a.access_type.or(a.alt_access_type), + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) + }) + .map(|a| a.path.clone()) + .collect(); + match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await { + Ok(edges) if !edges.is_empty() => { + append_logs( + job_id, + w_id, + format!( + "\nThese subscriptions will not fire — a dbt run does not trigger downstream \ + runs, and nothing else writes their relation:\n {}\n", + edges.join("\n ") + ), + conn, + ) + .await; + } + Ok(_) => {} + Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"), + } +} + /// Serialize publishers for one script path and confirm this job's version is /// still the newest. Both happen inside the caller's transaction, so a newer /// publisher either commits before this check sees it, or waits behind it and @@ -3411,6 +3933,12 @@ async fn resolve_selection( return Ok(None); } let mut cmd = dbt_command(p, &["ls"]); + // The same state the build resolves through, or a `result:` selector — which + // reads `run_results.json` out of it, and which `select` passes to dbt + // verbatim — fails here, before the build that would have honoured it. + if inv.deferral.is_some() { + cmd.args(defer_flags("ls", p.engine.engine)); + } // A project whose models call `var()` without a default fails to parse // without these, so the selection resolver needs them exactly as the run // does. Placeholders that only a run can fill are dropped rather than @@ -3424,11 +3952,15 @@ async fn resolve_selection( } cmd.args(["--output", "json", "--quiet"]); add_selection(&mut cmd, descriptor, inv)?; + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; // Captured directly, not through `handle_child`: its `pipe_stdout` path goes // through the job-log writer, which `NO_LOGS_AT_ALL` discards — the selection // would resolve to the empty set and the ingest would wipe the script's assets // while dbt went on building the descriptor's models. - let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES).await?; + let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES) + .await? + .stdout; let mut set = std::collections::HashSet::new(); for line in stdout.lines() { let line = line.trim(); @@ -3441,15 +3973,37 @@ async fn resolve_selection( } } } - if set.is_empty() { - // A selection that matches nothing would be ingested as "this script - // owns no relations", wiping its graph and cascade edges — the same - // outcome a failed capture produces, and indistinguishable from it. - // Refuse rather than silently un-wire the script. + // Empty is a real answer from a `state:` or `result:` method and from nothing + // else: `state:modified+` matches nothing exactly when nothing changed since + // the published state, and a run with no work to do is a successful one. Any + // other selection matching nothing is a selector that names nothing — a + // misspelled model, say — which must not pass as a build that did its job. + // Exempting by ORIGIN rather than by method would let every such typo through. + // + // What makes the exemption safe is that the empty set is never ingested as + // ownership, and that now holds through `check_state_selectors`: a `state:` + // or `result:` method survives it only from a run's OWN selection, which + // makes `add_caller_args` set `per_run_models`, which makes + // `publishes_ownership()` false, so the run stores a snapshot of its own. + // Relax the descriptor arm there and a descriptor-narrowed `state:modified+` + // reaches here on an unchanged project and wipes the graph the `else` below + // guards, with nothing failing. + if set.is_empty() && !selection_names(&select, &exclude, &["state", "result"]) { return Err(Error::ExecutionErr( - "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection rather \ - than deploying a script that owns nothing" - .to_string(), + if selection_is_overridden(descriptor, &inv.args)? { + "this run's `select`/`exclude` matched no dbt nodes, so it would build nothing; \ + check the selector. Only a `state:` or `result:` selector may match nothing, \ + its empty answer being a real one" + .to_string() + } else { + // The descriptor's is also ingested as "this script owns no + // relations", wiping its graph and cascade edges — the same + // outcome a failed capture produces, and indistinguishable from + // it. Refuse rather than silently un-wire the script. + "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection \ + rather than deploying a script that owns nothing" + .to_string() + }, )); } Ok(Some(set)) @@ -3459,6 +4013,34 @@ async fn resolve_selection( /// what is kept is the TAIL, because dbt prints its error summary last. const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; +/// What a captured invocation produced. `stderr` is where dbt writes its +/// diagnostics — the errors and warnings block — so a caller that has to explain +/// a SUCCESSFUL run needs it as much as a failing one does. +pub(crate) struct Captured { + pub stdout: String, + pub stderr: String, + /// Whether the child exited zero. Separate from the `Result` on purpose: an + /// `Err` from `run_captured` is the JOB's — a cancellation or its deadline — + /// so a caller that tolerates a failed command must still propagate one. + pub success: bool, + /// Whether the output ceiling cut the child short. Only ever true under + /// [`Overflow::Truncate`]. + pub truncated: bool, +} + +/// What an over-long stdout means to the caller. +/// +/// The ceiling belongs to the PASS, not to the job: a caller that only annotates +/// a job wants to keep what it read and carry on, while one whose whole result +/// is that output has nothing to return without it. +#[derive(PartialEq, Eq, Clone, Copy)] +pub(crate) enum Overflow { + /// Fail the job. For a command whose output IS the answer. + Fail, + /// Stop reading, kill the child, and report `truncated`. + Truncate, +} + /// Run a command for its stdout under the job's cancellation and timeout. /// The same poller `handle_child` uses drives them, so a cancel or a deadline /// drops the wait future — which owns the child, and `kill_on_drop` then @@ -3471,7 +4053,7 @@ const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; /// never holds more than it, so it has to be enforced while reading. Both pipes /// are drained concurrently because a child that fills the one nobody reads /// blocks forever. -async fn run_capturing( +pub(crate) async fn run_captured( mut cmd: Command, name: &str, ctx: &mut JobCtx<'_>, @@ -3479,7 +4061,8 @@ async fn run_capturing( w_id: &str, conn: &Connection, max_stdout_bytes: usize, -) -> error::Result { + on_overflow: Overflow, +) -> error::Result { use tokio::io::AsyncReadExt; let mut child = cmd @@ -3514,6 +4097,7 @@ async fn run_capturing( let mut out_buf = vec![0u8; 16 * 1024]; let mut err_buf = vec![0u8; 16 * 1024]; let (mut out_open, mut err_open) = (true, true); + let mut truncated = false; while out_open || err_open { tokio::select! { r = stdout_pipe.read(&mut out_buf[..]), if out_open => match r { @@ -3521,14 +4105,19 @@ async fn run_capturing( Ok(n) => { if stdout.len() + n > max_stdout_bytes { // Killed here rather than left to `kill_on_drop` - // so the child is gone before the error unwinds, - // not merely once this future is dropped. + // so the child is gone before this returns, not + // merely once the future is dropped. let _ = child.kill().await; - return Err(Error::ExecutionErr(format!( - "{name} produced more than {} MB of output. Narrow the \ - selection, or query the relation from a SQL script.", - max_stdout_bytes / 1024 / 1024 - ))); + if on_overflow == Overflow::Fail { + return Err(Error::ExecutionErr(format!( + "{name} produced more than {} MB of output. Narrow the \ + selection, or query the relation from a SQL script.", + max_stdout_bytes / 1024 / 1024 + ))); + } + truncated = true; + out_open = false; + continue; } stdout.extend_from_slice(&out_buf[..n]); } @@ -3551,7 +4140,7 @@ async fn run_capturing( .wait() .await .map_err(|e| Error::internal_err(format!("{name} failed: {e}")))?; - Ok((status, stdout, stderr)) + Ok((status, stdout, stderr, truncated)) }, ctx.worker_name, w_id, @@ -3561,14 +4150,46 @@ async fn run_capturing( })), ) .await?; - let (status, stdout, stderr) = out; - if !status.success() { + let (status, stdout, stderr, truncated) = out; + Ok(Captured { + stdout: String::from_utf8_lossy(&stdout).to_string(), + stderr: String::from_utf8_lossy(&stderr).to_string(), + // A killed child reports failure; under `Truncate` that is the ceiling's + // doing, not the project's, and the caller reads `truncated` to tell. + success: status.success(), + truncated, + }) +} + +/// `run_captured`, with a non-zero exit folded into the error — what a caller +/// that needs the command to have WORKED wants. +pub(crate) async fn run_capturing( + cmd: Command, + name: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + max_stdout_bytes: usize, +) -> error::Result { + let captured = run_captured( + cmd, + name, + ctx, + job_id, + w_id, + conn, + max_stdout_bytes, + Overflow::Fail, + ) + .await?; + if !captured.success { return Err(Error::ExecutionErr(format!( "{name} failed: {}", - String::from_utf8_lossy(&stderr) + captured.stderr ))); } - Ok(String::from_utf8_lossy(&stdout).to_string()) + Ok(captured) } /// Run a preparation command through the same child handler the build uses, so @@ -3761,7 +4382,7 @@ async fn save_run_state( if let Connection::Sql(db) = conn { { // Only while a live dbt version stays at this path — the test - // `clear_dbt_run_state_if_path_retired` retires state by, plus the + // `clear_dbt_script_state_if_path_retired` retires state by, plus the // language, since a rename leaves the old path archived rather than // deleted and a path can come back as another language. A job already // running finishes after those move or clear the row: writing then @@ -3923,6 +4544,12 @@ pub struct Invocation { /// what it pointed at must not. pub raw_args: HashMap>, pub envs: HashMap, + /// The stored dbt state this invocation resolves an unbuilt `ref()` through, + /// materialised into the job directory. Carried here rather than passed to + /// each phase: the model phase, the `after_all` tests and every in-job node + /// retry must all resolve a `ref()` the same way, or the tests assert against + /// relations the models never read. + pub deferral: Option, /// A run must fail on a `{{ }}` placeholder it cannot fill; a deploy, which /// has no arguments at all, tolerates them. Declared rather than inferred /// from the argument count: a run submitted with `{}` is still a run, and @@ -4075,11 +4702,12 @@ async fn restore_from_db( if !has_retryable_node(&row.run_results) { return Err(nothing_to_retry()); } - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - tokio::fs::write(target.join("run_results.json"), &row.run_results) - .await - .map_err(|e| Error::internal_err(format!("restoring run_results.json: {e}")))?; + write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&row.run_results), + StateManifest::None, + ) + .await?; // No manifest came with the row, so one has to be re-derived — but not here: // these arguments are as SUBMITTED, and a `$var:` in them shapes the graph // only once resolved. The caller resolves, then parses. @@ -4311,20 +4939,16 @@ async fn restore_run_state( return Err(different_project()); } let saved_args_digest = saved_args_digest.map(str::to_string); - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - // From the bytes already read, not by copying the file again: a burst of saves - // can prune this generation mid-restore, and a `dbt retry` whose + // The results go from the bytes already read, not by copying the file again: a + // burst of saves can prune this generation mid-restore, and a `dbt retry` whose // `run_results.json` went missing rebuilds nothing and reports success. The // manifest has no such copy, so a failure there falls back to a `dbt parse`. - tokio::fs::write(target.join("run_results.json"), &saved_results) - .await - .map_err(|e| { - Error::internal_err(format!("could not restore the previous run's results: {e}")) - })?; - let needs_parse = tokio::fs::copy(snapshot.join("manifest.json"), target.join("manifest.json")) - .await - .is_err(); + let needs_parse = !write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&saved_results), + StateManifest::CopyOf(snapshot.join("manifest.json")), + ) + .await?; // The generation was chosen from a row read before the file work above. A run // finishing in that window publishes a newer one, and resuming the superseded // generation redoes nodes it has already rebuilt — appending to an incremental @@ -4518,7 +5142,11 @@ fn has_retryable_node(run_results: &str) -> bool { } /// Append `--vars` if the descriptor (or the run) declares any. -fn add_vars(cmd: &mut Command, descriptor: &DbtDescriptor, inv: &Invocation) -> error::Result<()> { +pub(crate) fn add_vars( + cmd: &mut Command, + descriptor: &DbtDescriptor, + inv: &Invocation, +) -> error::Result<()> { let vars = resolved_vars(descriptor, &inv.args, inv.strict)?; if !vars.is_empty() { cmd.args(["--vars", &serde_json::to_string(&vars).unwrap_or_default()]); @@ -4713,10 +5341,26 @@ fn add_selection( descriptor: &DbtDescriptor, inv: &Invocation, ) -> error::Result<()> { - for s in effective_select(descriptor, inv)? { + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; + // The seam itself, which the DEPLOY reaches without going through a run: it + // resolves the descriptor's selection to decide what the script owns, and + // never computes a `defer`. A run has been checked earlier, where the message + // can still come before the state fetch. + check_state_selectors( + &select, + &exclude, + if inv.deferral.is_some() { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(descriptor, &inv.args)?, + )?; + for s in select { cmd.args(["--select", &s]); } - for s in effective_exclude(descriptor, inv)? { + for s in exclude { cmd.args(["--exclude", &s]); } if let Some(sel) = effective_selector(descriptor, inv)? { @@ -4725,6 +5369,115 @@ fn add_selection( Ok(()) } +/// The method a selection token names, with the graph operators that can +/// surround a node stripped (`@model`, `+model`, `2+model`, `model+`). +fn selector_method(token: &str) -> Option<&str> { + token + .trim_start_matches('@') + .trim_start_matches(|c: char| c.is_ascii_digit()) + .trim_start_matches('+') + .split_once(':') + .map(|(method, _)| method) +} + +/// Every method a selection names. Each entry is a union of whitespace-separated +/// tokens, and each of those an intersection of comma-separated ones. +fn selection_methods<'a>(entries: &'a [String]) -> impl Iterator { + entries + .iter() + .flat_map(|entry| entry.split([' ', '\t', ','])) + .filter_map(selector_method) +} + +/// Whether the run being checked has the state directory a `state:` or `result:` +/// method reads, or could be given one. +#[derive(Clone, Copy)] +enum StateAccess<'a> { + /// Deferring, so the directory is there. + Given, + /// Not deferring, and `defer` is what would hand it one. + OnRequest, + /// This command resolves a selection without ever deferring, so no setting + /// gives it a state and "turn `defer` on" would be advice that leads nowhere. + Never(&'a str), +} + +/// Whether a selection names any of these methods. +fn selection_names(select: &[String], exclude: &[String], methods: &[&str]) -> bool { + selection_methods(select) + .chain(selection_methods(exclude)) + .any(|method| methods.contains(&method)) +} + +/// Refuse a selection dbt cannot resolve, before it silently resolves to the +/// wrong thing. +/// +/// `state:` and `result:` compare against the artifacts in `--state`, which only +/// a deferring run is given. The engines do not agree on what happens without +/// one: dbt-core 1.x raises, but dbt-sa-cli and fusion read a missing state as an +/// EMPTY one and exit 0, so `state:modified` builds nothing and `state:new` +/// builds the whole project, each as a run that reports success. +/// +/// From the DESCRIPTOR they are refused whether or not the run defers, because +/// that selection also decides which nodes the script owns, and "whatever changed +/// last" is not an ownership answer — the deploy resolves it with no state at all. +/// They describe one run, so they belong in a run's own `select`. +/// +/// `source_status:` compares `sources.json`, which `dbt source freshness` writes +/// and no run publishes here, so it has nothing to compare against under any +/// setting. +/// +/// Only what `select` and `exclude` spell directly: a method reached through a +/// `selectors.yml` definition is named nowhere the worker can read, and dbt's +/// own behaviour is what stands there. +fn check_state_selectors( + select: &[String], + exclude: &[String], + access: StateAccess<'_>, + from_descriptor: bool, +) -> error::Result<()> { + for method in selection_methods(select).chain(selection_methods(exclude)) { + match method { + "source_status" => { + return Err(Error::BadRequest( + "a `source_status:` selector compares the source freshness recorded in \ + `sources.json`, which `dbt source freshness` writes and no run stores \ + here, so there is nothing for it to compare against. Drop the selector" + .to_string(), + )) + } + "state" | "result" if from_descriptor => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector describes what ONE run builds, but the descriptor's \ + selection also decides which nodes this script owns, which a deploy \ + resolves with no state to compare against. Move it to the `select` of a \ + run with `defer` on" + ))) + } + "state" | "result" => match access { + StateAccess::Given => {} + StateAccess::OnRequest => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and only a run with `defer` on is given \ + that state. Turn `defer` on, or drop the selector" + ))) + } + StateAccess::Never(command) => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and `{command}` resolves its selection \ + without building and never defers, so no setting hands it that state. \ + Drop the selector" + ))) + } + }, + _ => {} + } + } + Ok(()) +} + /// Whether this invocation chose its own `select`/`exclude`. /// /// DIFFERENT from the descriptor's, not merely present: `parse_dbt_sig` gives @@ -4748,6 +5501,41 @@ fn selection_is_overridden( Ok(differs("select", &descriptor.select)? || differs("exclude", &descriptor.exclude)?) } +/// Whether this invocation rebuilds incremental models from scratch: the run +/// form's answer when it gave one, else the descriptor's — and never for a +/// `test`, which builds nothing whatever the form said. +/// +/// Shared with the column-lineage pass rather than recomputed there, because +/// `is_incremental()` branches on it: the same model compiles to different SQL — +/// a `{{ this }}` self-join, and any `ref()` inside the incremental branch — so a +/// pass that guessed would describe a build that never ran. +/// +/// `test` returns false because `dbt test` rejects `--full-refresh` outright. +/// It never arrives as a caller's `dbt_command` — the allowlist has no such +/// value — so reading only that allowlist suggests this branch is dead. It is +/// not: `run_dbt` is invoked with `"test"` directly for the `after_all` test +/// phase, and an `after_all` project with `full_refresh: true` reaches here. +pub(crate) fn full_refresh( + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, +) -> error::Result { + if command == "test" { + return Ok(false); + } + Ok(arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh)) +} + +/// Whether this run answered `full_refresh` differently from the deployed +/// descriptor. Like a selection override it changes what the graph describes, +/// since an incremental branch can carry its own `ref()`. +fn full_refresh_is_overridden( + descriptor: &DbtDescriptor, + args: &HashMap>, +) -> error::Result { + Ok(arg_bool(args, "full_refresh")?.is_some_and(|v| v != descriptor.full_refresh)) +} + /// The descriptor's named selector, unless this run named its own selection. /// /// dbt resolves `--selector` INSTEAD of `--select`, so passing both makes the @@ -5007,20 +5795,47 @@ mod tests { // and without normalizing it the saved run is never recognized as its own. #[test] fn profile_identity_ignores_the_attempts_token() { - let yaml = |tok: &str| format!("host: \"wh\"\npassword: \"{tok}\"\n"); + let yaml = |v: &str| format!("host: \"{v}\"\nuser: \"u\"\n"); let dir = Path::new("/tmp/windmill/w/job-1/profiles"); assert_eq!( profile_identity_digest(&yaml("tok-first"), dir, None, "tok-first"), profile_identity_digest(&yaml("tok-retry"), dir, None, "tok-retry") ); - // A password that is NOT the job's token is the connection, and changing - // it must still read as a different warehouse. + // A value that is NOT the job's token is the connection, and changing it + // must still read as a different warehouse. assert_ne!( profile_identity_digest(&yaml("static-a"), dir, None, "tok-first"), profile_identity_digest(&yaml("static-b"), dir, None, "tok-retry") ); } + // Project code can leave a symlink where the worker later writes the profile: + // either the directory or the file in it. Neither may be followed. + #[cfg(unix)] + #[tokio::test] + async fn fresh_dir_never_follows_what_the_jail_left() { + let root = tempfile::tempdir().unwrap(); + let host = root.path().join("host"); + std::fs::create_dir(&host).unwrap(); + std::fs::write(host.join("profiles.yml"), "host file").unwrap(); + let dir = root.path().join("dbt_profiles"); + + std::os::unix::fs::symlink(&host, &dir).unwrap(); + fresh_dir(&dir).await.unwrap(); + assert!(!std::fs::symlink_metadata(&dir).unwrap().is_symlink()); + std::fs::write(dir.join("profiles.yml"), "rendered").unwrap(); + + fresh_dir(&dir).await.unwrap(); + std::os::unix::fs::symlink(host.join("profiles.yml"), dir.join("profiles.yml")).unwrap(); + fresh_dir(&dir).await.unwrap(); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); + + assert_eq!( + std::fs::read_to_string(host.join("profiles.yml")).unwrap(), + "host file" + ); + } + // The jail profile is protobuf text format, and the project path and the // descriptor's environment land inside string literals. An unescaped quote or // newline closes the literal and lets the rest be read as further directives — @@ -5738,6 +6553,79 @@ mod tests { .unwrap(); assert!(untouched.publishes_ownership()); assert_eq!(untouched.snapshot_job(job), None); + + // `resolve_selection` lets a selection match nothing on exactly this + // predicate, because a run that scoped its own selection stores a + // snapshot instead of publishing ownership. Should the two ever drift + // apart, an empty caller selection would wipe the script's graph and + // cascade edges, which is the outcome that guard exists to prevent. + // One-directional: a `vars` override also withholds ownership without + // touching the selection, which is why this is an implication and not an + // equivalence. + for args in [ + arg("select", r#"["state:modified+"]"#), + arg("exclude", r#"["tag:nightly"]"#), + ] { + assert!(selection_is_overridden(&descriptor, &args).unwrap()); + let mut g = GraphRefresh::default(); + g.add_caller_args(&descriptor, &args).unwrap(); + assert!( + !g.publishes_ownership(), + "an overridden selection must not publish ownership" + ); + } + + // `full_refresh` decides whether `is_incremental()` is true, so an + // incremental model's self-join — and any `ref()` inside that branch — + // exists in one answer and not the other. A run that flips it describes + // a different graph, and gets its own. + let mut refreshed = GraphRefresh::default(); + refreshed + .add_caller_args(&descriptor, &arg("full_refresh", "true")) + .unwrap(); + assert!(refreshed.needed()); + assert_eq!(refreshed.snapshot_job(job), Some(job)); + + // The same echo rule: the form posts the descriptor's own value back on + // every run, and reading that as an override would make each one + // caller-scoped. + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let mut echoed_flag = GraphRefresh { profile_drift: true, ..Default::default() }; + echoed_flag + .add_caller_args(&always, &arg("full_refresh", "true")) + .unwrap(); + assert_eq!(echoed_flag.snapshot_job(job), None); + } + + /// The build and the analysis pass read this through one function, so they + /// cannot disagree about which SQL the run compiles — including for `test`, + /// which rebuilds nothing whatever the descriptor or the form said. + #[test] + fn full_refresh_is_one_answer_for_the_build_and_the_pass() { + let inv = |args: HashMap>| Invocation { + args, + raw_args: Default::default(), + envs: Default::default(), + strict: true, + deferral: None, + }; + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let never = DbtDescriptor::default(); + let on = HashMap::from([( + "full_refresh".to_string(), + RawValue::from_string("true".to_string()).unwrap(), + )]); + + assert!(full_refresh(&always, &inv(Default::default()), "build").unwrap()); + assert!(!full_refresh(&never, &inv(Default::default()), "build").unwrap()); + assert!( + full_refresh(&never, &inv(on), "build").unwrap(), + "the form's answer wins over the descriptor's" + ); + assert!( + !full_refresh(&always, &inv(Default::default()), "test").unwrap(), + "a test builds nothing, so neither the build nor the pass may pass the flag" + ); } // `dbt retry` restores the previous run's target/ from this directory, so two @@ -5766,6 +6654,128 @@ mod tests { ); } + // A profile whose location dbt renders cannot be told apart from another + // rendering of itself, so it neither publishes state nor defers. Both + // delimiters count: a conditional block moves a schema exactly as an + // `env_var()` substitution does. + #[test] + fn a_rendered_profile_location_is_recognised_by_either_delimiter() { + assert!(is_jinja("{{ env_var('DBT_SCHEMA') }}")); + assert!(is_jinja( + "{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}" + )); + assert!(!is_jinja("analytics")); + assert!(!is_jinja("")); + } + + // dbt-sa-cli and fusion exit 0 on a state selector with no state, so nothing + // downstream would report this: the graph operators have to be stripped for + // the method to be seen at all. + #[test] + fn a_state_selector_is_found_under_any_graph_operator() { + // A run's own selection, which is the only place these belong. + let refused = |sel: &str, access: StateAccess<'_>| { + check_state_selectors(&[sel.to_string()], &[], access, false).is_err() + }; + for sel in [ + "state:modified", + "state:modified+", + "+state:new", + "@state:modified", + "2+state:modified+3", + "tag:nightly,state:modified", + "stg_orders+ result:error+", + ] { + assert!( + refused(sel, StateAccess::OnRequest), + "{sel} should need `defer`" + ); + assert!( + !refused(sel, StateAccess::Given), + "{sel} should pass while deferring" + ); + // A parse resolves a selection without ever deferring, so it is + // refused where a run would have been told to turn `defer` on. + assert!( + refused(sel, StateAccess::Never("parse")), + "{sel} cannot parse" + ); + // The descriptor's selection also decides what the script owns, and + // the deploy resolves it with no state, so deferring cannot save it. + assert!( + check_state_selectors(&[sel.to_string()], &[], StateAccess::Given, true).is_err(), + "{sel} should never be a descriptor selection" + ); + } + // A node whose name merely starts with a method's letters is not one. + for sel in ["stg_orders+", "tag:nightly", "stateful_model+"] { + assert!( + !refused(sel, StateAccess::OnRequest), + "{sel} is not a state selector" + ); + } + // No run publishes `sources.json`, so deferring does not help. + for access in [ + StateAccess::Given, + StateAccess::OnRequest, + StateAccess::Never("parse"), + ] { + assert!(refused("source_status:fresher+", access)); + } + // `exclude` reaches dbt the same way `select` does. + assert!(check_state_selectors( + &[], + &["state:modified".to_string()], + StateAccess::OnRequest, + false + ) + .is_err()); + + // The same recognition decides which empty selections `resolve_selection` + // lets through. Only these two answer "nothing" meaningfully; a selector + // naming nothing must not pass as a build that did its work. + const STATE_BACKED: &[&str] = &["state", "result"]; + for sel in ["state:modified+", "result:error+", "tag:x,state:new"] { + assert!( + selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + for sel in [ + "mispelled_model", + "tag:nightly", + "stg_orders+", + "source_status:fresher+", + ] { + assert!( + !selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + } + + // The one flag choice that is silently wrong rather than loudly wrong: a + // `retry` handed `--state` resumes the SUCCESSFUL run stored there and + // rebuilds nothing, reporting a green retry of a failed run. + #[test] + fn a_retry_is_never_handed_the_deferral_as_its_state() { + assert_eq!( + defer_flags("build", DbtEngine::DbtCore1x), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("test", DbtEngine::Fusion), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("retry", DbtEngine::DbtCore1x), + ["--defer-state", crate::dbt_state::STATE_DIR] + ); + for engine in [DbtEngine::DbtCore2x, DbtEngine::Fusion] { + assert!(defer_flags("retry", engine).is_empty()); + } + } + #[test] fn events_without_a_relation_are_not_materializations() { // A test node has no relation of its own. diff --git a/backend/windmill-worker/src/dbt_profiles.rs b/backend/windmill-worker/src/dbt_profiles.rs index 3d96f84676..f81aea2ae5 100644 --- a/backend/windmill-worker/src/dbt_profiles.rs +++ b/backend/windmill-worker/src/dbt_profiles.rs @@ -7,12 +7,42 @@ use serde_json::Value; use windmill_common::error::{self, Error}; +use windmill_parser_yaml::dbt::DbtEngine; /// Written beside `profiles.yml`, and named absolutely in `sslrootcert`: dbt /// runs with the project as its working directory and hands the path to the /// driver unchanged. pub const ROOT_CERT_FILENAME: &str = "server-ca.pem"; +/// What a credential's value becomes in [`RenderedProfile::identity`]. +const MASKED_CREDENTIAL: &str = "$CREDENTIAL"; + +/// Whether a target key holds a credential rather than part of the address. By +/// name, because a `dbt_profile` block's keys are its adapter's own: `password`, +/// dbt-postgres's `pass`, `token`, `private_key_passphrase`, `client_secret`, +/// `aws_secret_access_key` and the `key_id` rotated with it, … An endpoint is +/// address even when its name says otherwise: BigQuery's `token_uri` is where the +/// token comes from. +fn is_credential_key(key: &str) -> bool { + let key = key.to_ascii_lowercase(); + if key.ends_with("_uri") || key.ends_with("_url") || key.contains("endpoint") { + return false; + } + key == "pass" + || [ + "password", + "passphrase", + "secret", + "token", + "private_key", + "api_key", + "access_key", + "key_id", + ] + .iter() + .any(|c| key.contains(c)) +} + /// The per-adapter facts, so each adapter states them together and a new one /// cannot inherit another's by omission. `PG` is the base every arm spreads /// from: most adapters differ from Postgres only in their name and package. @@ -469,6 +499,10 @@ fn port_of(resource: &Value, default: i64) -> error::Result { #[derive(Debug)] pub struct RenderedProfile { pub yaml: String, + /// `yaml` with every credential's value masked: what run identity hashes. A + /// rotated token or password is still the same connection, and an OAuth token + /// rotates every few minutes, so hashing it would refuse nearly every retry. + pub identity: String, pub schema: Option, pub database: Option, /// A private CA the caller must write next to `profiles.yml`, under the @@ -483,8 +517,11 @@ pub struct RenderedProfile { /// from the descriptor when set, else from the resource, else from dbt's own /// per-adapter default — dbt errors out clearly when it ends up missing, which /// is a better failure than a Windmill-invented default. +#[allow(clippy::too_many_arguments)] pub fn render_profile( adapter: &DbtAdapter, + // The engines read some keys differently; see the Snowflake token below. + engine: DbtEngine, resource: &Value, profile_name: &str, target: &str, @@ -593,11 +630,18 @@ pub fn render_profile( out.push(("user".into(), quoted(&u))); } // Key-pair is Windmill's own snowflake resource shape; the - // `snowflake_oauth` type carries a token instead, which dbt only - // accepts alongside `authenticator: oauth` — without both, the - // profile renders with no credential at all and cannot connect. + // `snowflake_oauth` type carries an access token instead, which needs + // an `authenticator` saying so. dbt-core 1.x reads `oauth` + `token` + // as one. The Rust engines read `oauth` as the refresh-token flow and + // refuse a profile without client credentials, and send the same + // login for `jwt`, which dbt-snowflake has only from 1.9, while the + // 1.x engine still resolves 1.8. if let Some(t) = s(resource, "token").or_else(|| s(resource, "access_token")) { - out.push(("authenticator".into(), quoted("oauth"))); + let authenticator = match engine { + DbtEngine::DbtCore1x => "oauth", + DbtEngine::DbtCore2x | DbtEngine::Fusion => "jwt", + }; + out.push(("authenticator".into(), quoted(authenticator))); out.push(("token".into(), quoted(&t))); } else if let Some(k) = s(resource, "private_key") { out.push(("private_key".into(), quoted(&k))); @@ -612,10 +656,17 @@ pub fn render_profile( out.push((k.into(), quoted(&v))); } } - database = s(resource, "database"); - if let Some(d) = database.clone() { - out.push(("database".into(), quoted(&d))); - } + // dbt-snowflake requires one, and a resource from the OAuth connect flow + // starts without it: naming the field beats dbt's schema error. + let db = s(resource, "database").ok_or_else(|| { + Error::BadRequest( + "a Snowflake target needs a database; add `database` to the warehouse's \ + resource" + .to_string(), + ) + })?; + out.push(("database".into(), quoted(&db))); + database = Some(db); schema = schema.or_else(|| s(resource, "schema")); } KnownAdapter::Bigquery => { @@ -692,24 +743,43 @@ pub fn render_profile( // a newline in one opens a sibling key of the caller's choosing. let (qp, qt) = (yaml_scalar(profile_name), yaml_scalar(target)); let mut yaml = format!("{qp}:\n target: {qt}\n outputs:\n {qt}:\n"); + let mut identity = yaml.clone(); for (k, v) in &out { yaml.push_str(&format!(" {k}: {}\n", v.render())); + let shown = if is_credential_key(k) { + yaml_scalar(MASKED_CREDENTIAL) + } else { + v.render() + }; + identity.push_str(&format!(" {k}: {shown}\n")); } // The service-account document is a nested mapping, not a scalar. if adapter == KnownAdapter::Bigquery { yaml.push_str(" keyfile_json:\n"); + identity.push_str(" keyfile_json:\n"); let obj = resource .as_object() .ok_or_else(|| Error::BadRequest("bigquery resource is not an object".to_string()))?; for (k, v) in obj { if let Some(v) = v.as_str() { yaml.push_str(&format!(" {}: {}\n", yaml_scalar(k), yaml_scalar(v))); + let shown = if is_credential_key(k) { + MASKED_CREDENTIAL + } else { + v + }; + identity.push_str(&format!( + " {}: {}\n", + yaml_scalar(k), + yaml_scalar(shown) + )); } } } Ok(RenderedProfile { yaml, + identity, schema, database, root_certificate_pem: matches!(adapter, KnownAdapter::Postgres) @@ -746,6 +816,7 @@ pub fn render_dbt_profile( " \"type\": {}\n", yaml_scalar(adapter.dbt_type()) )); + let mut identity = yaml.clone(); for (k, v) in block { // A null is an optional field the resource form left unset, and dbt // validates several keys against a schema that rejects one. @@ -762,28 +833,33 @@ pub fn render_dbt_profile( if (k == schema_key && schema_override.is_some()) || (k == "threads" && threads.is_some()) { continue; } - emit_entry(&mut yaml, 6, k, v); + emit_entry(&mut yaml, 6, k, v, false); + emit_entry(&mut identity, 6, k, v, true); } + let mut tail = String::new(); if root_certificate_pem.is_some() { - yaml.push_str(&format!( + tail.push_str(&format!( " \"sslrootcert\": {}\n", yaml_scalar(&profiles_dir.join(ROOT_CERT_FILENAME).to_string_lossy()) )); } if let Some(sc) = schema_override { - yaml.push_str(&format!( + tail.push_str(&format!( " {}: {}\n", yaml_scalar(schema_key), yaml_scalar(sc) )); } if let Some(t) = threads { - yaml.push_str(&format!(" \"threads\": {t}\n")); + tail.push_str(&format!(" \"threads\": {t}\n")); } + yaml.push_str(&tail); + identity.push_str(&tail); let str_key = |k: &str| block.get(k).and_then(|v| v.as_str()).map(|v| v.to_string()); Ok(RenderedProfile { yaml, + identity, schema: schema_override .map(|x| x.to_string()) .or_else(|| str_key(schema_key)), @@ -794,16 +870,21 @@ pub fn render_dbt_profile( /// Emit one target key, nesting as deep as the value goes — an adapter's credential can be /// a mapping (bigquery's `keyfile_json`) or a list. Keys are quoted like values: one nothing -/// here enumerates is as free-form as a password. -fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value) { +/// here enumerates is as free-form as a password. `mask` writes credentials as +/// [`MASKED_CREDENTIAL`], at any depth, for [`RenderedProfile::identity`]. +fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value, mask: bool) { out.push_str(&format!("{}{}:", " ".repeat(indent), yaml_scalar(key))); - emit_value(out, indent, v); + emit_value(out, indent, v, mask, mask && is_credential_key(key)); } /// The value half, after `key:`. An empty collection is emitted INLINE: a block with no /// children reads back as `null`, so `extensions: []` would reach the adapter as a missing /// value rather than the empty list dbt was handed. -fn emit_value(out: &mut String, indent: usize, v: &Value) { +/// +/// `credential` masks scalars only. A collection under a credential-named key is still +/// walked: dbt-duckdb's `secrets:` list holds the endpoint and scope that say which +/// connection it is, each entry judged by its own key. +fn emit_value(out: &mut String, indent: usize, v: &Value, mask: bool, credential: bool) { match v { Value::Object(m) => { // A null is an optional field the resource form left unset, and dbt validates @@ -815,7 +896,7 @@ fn emit_value(out: &mut String, indent: usize, v: &Value) { } out.push('\n'); for (k, v) in kept { - emit_entry(out, indent + 2, k, v); + emit_entry(out, indent + 2, k, v, mask); } } Value::Array(items) => { @@ -828,9 +909,10 @@ fn emit_value(out: &mut String, indent: usize, v: &Value) { for item in items { out.push_str(&pad); out.push('-'); - emit_value(out, indent + 2, item); + emit_value(out, indent + 2, item, mask, credential); } } + _ if credential => out.push_str(&format!(" {}\n", yaml_scalar(MASKED_CREDENTIAL))), _ => out.push_str(&format!(" {}\n", yaml_value(v))), } } @@ -904,6 +986,7 @@ mod tests { "dbname": "warehouse", "sslmode": "require"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -936,6 +1019,7 @@ mod tests { "password": "p", "dbname": "warehouse"}); let p = render_profile( &KnownAdapter::Redshift.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1136,6 +1220,7 @@ mod tests { "http_path": "/sql/1.0/warehouses/x", "token": "t"}); let p = render_profile( &KnownAdapter::Databricks.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1161,6 +1246,7 @@ mod tests { "root_certificate_pem": "-----BEGIN CERTIFICATE-----\nx\n"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1188,6 +1274,7 @@ mod tests { let plain = json!({"host": "h", "dbname": "d", "sslmode": "require"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &plain, "wm", "prod", @@ -1200,29 +1287,107 @@ mod tests { assert_eq!(p.root_certificate_pem, None); } - // `snowflake_oauth` maps to the Snowflake adapter, but its credential is a - // token, which dbt honors only with `authenticator: oauth`. Forwarding neither - // renders a profile with no credential at all. - #[test] - fn snowflake_oauth_renders_its_token() { - let r = json!({"account_identifier": "acc", "username": "u", "token": "tok", - "database": "db", "warehouse": "wh"}); - let p = render_profile( + fn snowflake(engine: DbtEngine, r: &Value) -> error::Result { + render_profile( &KnownAdapter::Snowflake.into(), - &r, + engine, + r, "wm", "prod", None, None, std::path::Path::new("/tmp/p"), ) - .unwrap(); - assert!( - p.yaml.contains(" authenticator: \"oauth\"\n"), - "{}", - p.yaml - ); - assert!(p.yaml.contains(" token: \"tok\"\n")); + } + + // `snowflake_oauth` carries an access token, which only one `authenticator` + // per engine accepts: the Rust engines refuse `oauth` without client + // credentials, and dbt-snowflake before 1.9 has no `jwt`. + #[test] + fn snowflake_oauth_names_its_token_per_engine() { + let r = json!({"account_identifier": "acc", "token": "tok", "database": "db"}); + for (engine, authenticator) in [ + (DbtEngine::DbtCore1x, "oauth"), + (DbtEngine::DbtCore2x, "jwt"), + (DbtEngine::Fusion, "jwt"), + ] { + let p = snowflake(engine, &r).unwrap(); + assert!( + p.yaml + .contains(&format!(" authenticator: \"{authenticator}\"\n")), + "{engine:?}: {}", + p.yaml + ); + assert!(p.yaml.contains(" token: \"tok\"\n")); + } + let err = snowflake( + DbtEngine::DbtCore1x, + &json!({"account_identifier": "acc", "token": "tok"}), + ) + .unwrap_err() + .to_string(); + assert!(err.contains("add `database`"), "{err}"); + } + + // Run identity has to survive a credential rotating, which an OAuth token does + // every few minutes, and still change when the connection moves. + #[test] + fn identity_masks_credentials_but_not_the_connection() { + let rendered = |account: &str, token: &str| { + snowflake( + DbtEngine::DbtCore1x, + &json!({"account_identifier": account, "token": token, "database": "db"}), + ) + .unwrap() + .identity + }; + assert_eq!(rendered("acc", "t1"), rendered("acc", "t2")); + assert_ne!(rendered("acc", "t1"), rendered("other", "t1")); + + // A `dbt_profile` block, whose keys are the adapter's own, nested ones too. + let block = |adapter: KnownAdapter, v: Value| { + render_dbt_profile( + &adapter.into(), + v.as_object().unwrap(), + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap() + .identity + }; + let bq = |project: &str, key: &str, token_uri: &str| { + block( + KnownAdapter::Bigquery, + json!({"type": "bigquery", "project": project, "dataset": "d", + "keyfile_json": {"client_email": "e", "private_key": key, + "token_uri": token_uri}}), + ) + }; + assert_eq!(bq("p", "k1", "t"), bq("p", "k2", "t")); + assert_ne!(bq("p", "k1", "t"), bq("q", "k1", "t")); + assert_ne!(bq("p", "k1", "t"), bq("p", "k1", "elsewhere")); + // Only scalars are masked: a `secrets:` entry still names its endpoint. + // A rotated access key changes its id with its secret. + let duck = |endpoint: &str, secret: &str| { + block( + KnownAdapter::Duckdb, + json!({"type": "duckdb", "path": "x.duckdb", + "secrets": [{"type": "s3", "endpoint": endpoint, + "key_id": format!("id-{secret}"), "secret": secret}]}), + ) + }; + assert_eq!(duck("s3.a", "k1"), duck("s3.a", "k2")); + assert_ne!(duck("s3.a", "k1"), duck("s3.b", "k1")); + let pg = |pass: &str| { + block( + KnownAdapter::Postgres, + json!({"type": "postgres", "host": "h", "user": "u", "pass": pass}), + ) + }; + assert_eq!(pg("p1"), pg("p2")); } // dbt rejects a BigQuery target with no dataset and a service-account JSON @@ -1233,6 +1398,7 @@ mod tests { let r = json!({"project_id": "p", "client_email": "e", "private_key": "k"}); let err = render_profile( &KnownAdapter::Bigquery.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1245,6 +1411,7 @@ mod tests { assert!(err.contains("profile.schema"), "{err}"); let p = render_profile( &KnownAdapter::Bigquery.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1284,6 +1451,7 @@ mod tests { let r = json!({"host": "h", "dbname": "sales", "user": "u"}); let p = render_profile( &KnownAdapter::Mysql.into(), + DbtEngine::DbtCore1x, &r, "wm", "dev", @@ -1304,6 +1472,7 @@ mod tests { fn a_profile_name_or_target_cannot_open_a_sibling_key() { let rendered = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &serde_json::json!({"host": "h", "user": "u", "password": "p", "dbname": "d"}), "prod # hidden", "dev\n evil: yes", @@ -1339,6 +1508,7 @@ mod tests { "password": "p\"\nhost: evil.example.com\n#"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "dev", diff --git a/backend/windmill-worker/src/dbt_state.rs b/backend/windmill-worker/src/dbt_state.rs new file mode 100644 index 0000000000..ecf3d0dbe3 --- /dev/null +++ b/backend/windmill-worker/src/dbt_state.rs @@ -0,0 +1,748 @@ +//! The dbt state a project last built into one environment, and the state +//! directory a run reads it back through. +//! +//! `dbt --defer --state ` resolves a `ref()` the run does not build to the +//! relation the manifest in `` names, instead of to the schema this run +//! writes into. That makes the state a durable, per-environment artifact rather +//! than a cache: the next run of a project usually lands on a worker holding +//! neither the manifest nor the results, so anything worker-local answers for +//! one machine's history rather than for the environment. +//! +//! Two artifacts live in that directory and both are stored: `manifest.json`, +//! which is what a deferral resolves through, and `run_results.json`, which +//! `select`'s `result:` selectors read — and `select` reaches dbt verbatim, so a +//! state directory missing it fails a selection a user may legitimately write. + +use std::path::{Path, PathBuf}; + +use uuid::Uuid; +use windmill_common::error::{self, Error}; +use windmill_common::worker::Connection; + +use crate::dbt_executor::{digest, PreparedProject, ARTIFACTS_DIR}; + +lazy_static::lazy_static! { + /// Above this, an artifact goes to the instance's object storage instead of + /// into the row. A manifest passes a few hundred KB on a handful of models + /// and grows with the project, so this ceiling is what decides whether a + /// large project needs storage configured at all; a small one stays in the + /// database, where it costs no round trip and needs nothing configured. + static ref DBT_STATE_INLINE_MAX_BYTES: usize = std::env::var("DBT_STATE_INLINE_MAX_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8 * 1024 * 1024); +} + +/// The directory `--state` points at. Inside the job directory, so it sits in +/// the sandbox's one writable bind and goes away with the job, and prefixed like +/// the artifacts directory beside it so a project carrying a directory of this +/// name is not overwritten. +/// +/// Passed to dbt RELATIVE, and that is load-bearing rather than tidiness. dbt +/// records the invocation's flags into `run_results.json` and a later +/// `dbt retry` restores them, so an absolute path would name the job directory +/// of the run being resumed — gone by then, leaving the retry to resolve a +/// deferred `ref()` against nothing. Relative, it resolves against the project +/// root, which is whichever job directory the retry landed in. +pub(crate) const STATE_DIR: &str = "wm_dbt_state"; + +/// Where this run's relations live, which is the only thing a deferral is about. +pub(crate) fn environment(p: &PreparedProject) -> String { + environment_key( + p.warehouse.as_deref(), + // The target dbt RUNS, not the descriptor's: it falls back to the + // workspace warehouse's and to the project's own default, so reading the + // descriptor's would put two inherited targets under one empty name. + p.effective_target.as_deref(), + // The pair `relation_root` reports to the graph's drift check, taken + // apart so neither can absorb the other's delimiter below. + p.default_schema.as_deref(), + p.default_database.as_deref(), + ) +} + +/// The warehouse and the target name the environment; the database and schema +/// they resolve to are in the key because a repointed warehouse resource or a +/// moved schema keeps both names while putting the relations somewhere else — +/// and a manifest is a list of relation names, so a deferral has no other way to +/// notice. A move therefore reads as an environment nothing has published yet. +/// +/// Length-prefixed rather than joined on a separator. Every component but the +/// warehouse is spelled by the user — a dbt target name and a schema are both +/// arbitrary strings a profile may quote — so `prod|analytics` + `scratch` and +/// `prod` + `analytics|scratch` would otherwise be one key, and a profile moving +/// between them would read as the same environment rather than as one nothing +/// has published. Same reasoning as `stable_digest`, and still legible in a row: +/// `4:main|4:prod|9:analytics|12:dbt_wh_defer`. What a MESSAGE names is +/// `environment_label`, since this encoding is for storage. +fn environment_key( + warehouse: Option<&str>, + target: Option<&str>, + schema: Option<&str>, + database: Option<&str>, +) -> String { + [warehouse, target, schema, database] + .iter() + .map(|v| { + let v = v.unwrap_or(""); + format!("{}:{v}", v.len()) + }) + .collect::>() + .join("|") +} + +/// The environment as a message names it: the key above is length-prefixed for +/// storage, which is not something to put in front of a caller. +pub(crate) fn environment_label(p: &PreparedProject) -> String { + format!( + "warehouse `{}`, target `{}`, relations in `{}`", + p.warehouse.as_deref().unwrap_or("(none)"), + p.effective_target + .as_deref() + .unwrap_or("(the profile's default)"), + match (p.default_database.as_deref(), p.default_schema.as_deref()) { + (Some(db), Some(schema)) => format!("{db}.{schema}"), + (None, Some(schema)) => schema.to_string(), + _ => "(the adapter's default)".to_string(), + } + ) +} + +/// The state one environment last published. +pub(crate) struct StoredState { + pub manifest: String, + pub run_results: Option, + /// The run that published it, so a deferring run can say what it deferred to. + pub job_id: Uuid, +} + +/// Publish this run's artifacts as the environment's state. +/// +/// Called for a run that BUILT what the script's own descriptor selects and +/// succeeded (see `handle_dbt_job`). Best-effort in the same sense as the retry +/// state: losing it costs the next deferral, not the run that just finished. +/// +/// **What the artifacts may carry follows from that condition.** A publishing run +/// added nothing of its own — no `select` or `vars` override, and a descriptor +/// interpolating a `{{ }}` placeholder into `vars` never publishes at all — so +/// dbt's `run_results.json` records the descriptor's own arguments, which are the +/// script's content. That is why this is keyed by environment where +/// `dbt_run_state` is keyed by principal: the retry state holds whatever a caller +/// submitted, this holds what the script says. Widen the publish condition and +/// that stops being true. +pub(crate) async fn publish( + p: &PreparedProject, + w_id: &str, + job_id: &Uuid, + // The version this job ran. `None` for a preview, which publishes nothing. + script_hash: Option, + // A build recovered by the automatic in-job node retry has a + // `run_results.json` naming only the nodes that retry redid. The manifest is + // unaffected — it is a function of the project, not of what ran — so the + // state is published without results rather than with a set describing some + // other slice of the build. + results_are_partial: bool, + conn: &Connection, +) -> error::Result<()> { + let Connection::Sql(db) = conn else { + // An agent worker reaches the database only through the API, which does + // not expose this table. + return Ok(()); + }; + if p.script_path.is_empty() { + // A preview has no path to key state on, and an empty one would be + // shared by every dbt script in the workspace. + return Ok(()); + } + if p.templated_location { + // Refused on this side too, not only where a deferral reads. A template + // renders to one location per environment while the key sees the + // template, so publishing would file this run's manifest under a key a + // literal profile shares — and de-templating later would make that stale + // manifest readable as the new location's. + return Ok(()); + } + let artifacts = p.project_dir.join(ARTIFACTS_DIR); + // The manifest is what a deferral resolves through, so there is no state + // without one. Every engine writes it beside the results of a build, so this + // is the invocation that built nothing rather than a case to report. + let Ok(manifest) = tokio::fs::read_to_string(artifacts.join("manifest.json")).await else { + return Ok(()); + }; + let run_results = match results_are_partial { + true => None, + false => tokio::fs::read_to_string(artifacts.join("run_results.json")) + .await + .ok(), + }; + let environment = environment(p); + // Uploaded BEFORE the transaction, and to this publication's own keys, so two + // publishers cannot collide on them and nothing here can overwrite an + // artifact a committed row still names. A failure below has only its own + // objects to drop. + let nonce = Uuid::new_v4(); + let (manifest, manifest_key) = store( + manifest, + "manifest.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await?; + let (run_results, run_results_key) = match run_results { + Some(r) => match store( + r, + "run_results.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await + { + Ok(stored) => stored, + Err(e) => { + forget_objects(&[manifest_key, None]).await; + return Err(e); + } + }, + None => (None, None), + }; + let mine = [manifest_key.clone(), run_results_key.clone()]; + // One publisher per environment at a time, so the row and the objects it + // displaces are settled by one of them at a time. An advisory lock rather + // than the row's, because the first publish of an environment has no row to + // lock and is exactly when two runs of a newly deployed script are most + // likely to race. + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + forget_objects(&mine).await; + return Err(e.into()); + } + }; + let staged = async { + sqlx::query_scalar!( + "SELECT pg_advisory_xact_lock($1)", + publication_lock(w_id, &p.script_path, &environment) + ) + .execute(&mut *tx) + .await?; + // The script row FIRST, and held, so a rename, archive or delete of this + // path either waits for this publication or is seen by it. Reading it + // unlocked leaves a window where lifecycle cleanup finds no row to clear, + // finishes, and this transaction then commits state at a path a new + // script goes on to occupy. Script row before sidecar is also the order + // every other dbt writer takes, which is what keeps the two off a + // deadlock. + // + // The version, not just the path: "some live dbt script is here" is also + // satisfied by a script created at a path this one was renamed away from. + // A preview names no version, so `script_hash` is NULL and nothing + // matches — right for a run of content that was never deployed. + let owns_path = sqlx::query_scalar!( + "SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false AND language = 'dbt' + AND (hash = $3 OR $3 = ANY(parent_hashes)) + FOR SHARE", + w_id, + &p.script_path, + script_hash, + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + if !owns_path { + return error::Result::Ok(None); + } + // What the row points at NOW, so those objects can go once this one is + // committed in their place — never before, since a reader that has + // already read the row is about to fetch them. + let displaced = sqlx::query!( + "SELECT manifest_key, run_results_key FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(&mut *tx) + .await? + .map(|r| [r.manifest_key, r.run_results_key]) + .unwrap_or_default() + // Never a key this publication is about to commit. The keys carry a + // per-execution nonce so the two cannot coincide, and this is what says + // so rather than leaving it to be re-derived. + .map(|k| k.filter(|k| !mine.iter().flatten().any(|m| m == k))); + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest, manifest_key, run_results, + run_results_key, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now()) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET + job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest, + manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results, + run_results_key = EXCLUDED.run_results_key, updated_at = now()", + w_id, + &p.script_path, + environment, + job_id, + manifest, + manifest_key, + run_results, + run_results_key, + ) + .execute(&mut *tx) + .await?; + error::Result::Ok(Some(displaced)) + } + .await; + let displaced = match staged { + // Refused by the guard, or the write failed: nothing is committed and + // what was uploaded above has no row naming it. + Ok(None) | Err(_) => { + forget_objects(&mine).await; + return staged.map(|_| ()); + } + Ok(Some(displaced)) => displaced, + }; + // A commit that reports an error may still have committed — what was lost can + // be the acknowledgement. Dropping this run's objects would then leave the + // committed row naming objects that are gone, and every deferral would fail + // until the next publication; an orphan costs storage instead. + tx.commit().await?; + forget_objects(&displaced).await; + Ok(()) +} + +/// The environment's state, or `None` where nothing has published one. +pub(crate) async fn load( + p: &PreparedProject, + w_id: &str, + conn: &Connection, +) -> error::Result> { + let Connection::Sql(db) = conn else { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the dbt state stored for this environment, which \ + an agent worker cannot read: it reaches the database only through the API. Run this \ + script on a worker of the main group, or without `defer`" + .to_string(), + )); + }; + let environment = environment(p); + // A publication committing between the row and the objects it named drops + // those objects, so a miss is re-read rather than reported as a state that is + // not there. Re-read for as long as the row keeps MOVING: a reader takes no + // lock, so back-to-back publications can each overtake it, and a fixed one + // retry would report the second as missing. An unmoved row is the other + // answer — nothing republished, so the object really is gone. + let mut tried: Option<(Uuid, Option, Option)> = None; + for _ in 0..PUBLICATIONS_OUTRUN { + let Some(row) = sqlx::query!( + "SELECT job_id, manifest, manifest_key, run_results, run_results_key + FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(db) + .await? + else { + return Ok(None); + }; + let seen = ( + row.job_id, + row.manifest_key.clone(), + row.run_results_key.clone(), + ); + let fetched = async { + let manifest = fetch(row.manifest, row.manifest_key).await?; + let run_results = fetch(row.run_results, row.run_results_key).await?; + error::Result::Ok((manifest, run_results)) + } + .await; + match fetched { + Ok((Some(manifest), run_results)) => { + return Ok(Some(StoredState { manifest, run_results, job_id: seen.0 })) + } + Ok((None, _)) => return Ok(None), + Err(e) => { + if tried.as_ref() == Some(&seen) { + return Err(e); + } + tried = Some(seen); + } + } + } + Err(Error::internal_err( + "the dbt state for this environment was replaced faster than it could be read; run this \ + script again" + .to_string(), + )) +} + +/// How many publications a read may lose to before it gives up. Each one costs a +/// re-read, and a project publishing this often while another run defers is +/// already contending for the same relations. +const PUBLICATIONS_OUTRUN: usize = 5; + +/// A `manifest.json` for a state directory, whichever side it comes from. +/// +/// One enum because the three restores — a deferral's stored state, a retry's +/// worker-local generation, a retry's database row — differ only in where the +/// bytes are, and a second copy of the directory layout is a second chance for +/// one of them to write a directory dbt reads differently. +pub(crate) enum StateManifest { + Bytes(String), + /// A file on this worker, copied rather than read into memory: a manifest + /// grows with the project. + CopyOf(PathBuf), + None, +} + +/// Write the artifacts a dbt state directory holds, creating it if needed. +/// +/// Returns whether a `manifest.json` ended up there — a worker-local generation +/// can be pruned out from under a restore, and the caller then owes a +/// `dbt parse` for one. +pub(crate) async fn write_state_dir( + dir: &Path, + run_results: Option<&str>, + manifest: StateManifest, +) -> error::Result { + tokio::fs::create_dir_all(dir) + .await + .map_err(|e| Error::internal_err(format!("preparing the dbt state directory: {e}")))?; + if let Some(run_results) = run_results { + tokio::fs::write(dir.join("run_results.json"), run_results) + .await + .map_err(|e| Error::internal_err(format!("writing run_results.json: {e}")))?; + } + Ok(match manifest { + StateManifest::Bytes(m) => { + tokio::fs::write(dir.join("manifest.json"), m) + .await + .map_err(|e| Error::internal_err(format!("writing manifest.json: {e}")))?; + true + } + StateManifest::CopyOf(from) => tokio::fs::copy(from, dir.join("manifest.json")) + .await + .is_ok(), + StateManifest::None => false, + }) +} + +/// The advisory lock one environment's publishers take, so only one of them +/// settles the row and the objects it displaces at a time. +/// +/// Derived from the same three components as the row's key. Two environments +/// whose digests collide wait for each other, which costs a moment and nothing +/// else. +fn publication_lock(w_id: &str, script_path: &str, environment: &str) -> i64 { + let d = digest(&format!("{w_id}|{script_path}|{environment}")); + // Parsed unsigned and reinterpreted: half of all digests set the top bit, + // and read as `i64` those overflow and would collapse onto one key. + u64::from_str_radix(&d[..16], 16).unwrap_or_default() as i64 +} + +/// The object-storage key an artifact takes. +/// +/// One key per PUBLICATION, so an upload never overwrites an artifact the +/// committed row still names: a run that fails between its two uploads, or +/// between them and its row, leaves the state pointing at the pair it already +/// had. The row switches to these in one statement and the objects it displaced +/// are dropped afterwards. The path and environment are only a prefix — the row +/// is what says where an artifact is, so state that moves with a renamed script +/// keeps naming objects under the old one. Digested because a Windmill path and a +/// schema name may both carry characters an object key gives meaning to. +/// +/// The `nonce` is per EXECUTION rather than per job, because zombie recovery +/// re-runs a job under its own id: keyed on that alone, the second attempt would +/// overwrite the objects the first attempt's committed row still names, and then +/// read those same keys back as displaced and drop them. +fn object_key( + w_id: &str, + script_path: &str, + environment: &str, + job_id: &Uuid, + nonce: &Uuid, + artifact: &str, +) -> String { + format!( + "wmill_dbt_state/{w_id}/{}/{job_id}.{nonce}/{artifact}", + digest(&format!("{script_path}|{environment}")) + ) +} + +/// Put an artifact where its size says it belongs: `(inline, key)`, exactly one +/// of which is set. +#[allow(clippy::too_many_arguments)] +async fn store( + value: String, + artifact: &str, + environment: &str, + script_path: &str, + w_id: &str, + job_id: &Uuid, + nonce: &Uuid, +) -> error::Result<(Option, Option)> { + if value.len() <= *DBT_STATE_INLINE_MAX_BYTES { + return Ok((Some(value), None)); + } + let key = object_key(w_id, script_path, environment, job_id, nonce, artifact); + let size = value.len(); + if put_object(&key, value).await? { + return Ok((None, Some(key))); + } + Err(Error::BadRequest(format!( + "this project's {artifact} is {}, past the {} this instance keeps in the database, and \ + this instance has no object storage configured to hold it. Configure instance object \ + storage, or raise DBT_STATE_INLINE_MAX_BYTES", + mib(size), + mib(*DBT_STATE_INLINE_MAX_BYTES), + ))) +} + +fn mib(bytes: usize) -> String { + format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0)) +} + +/// Read an artifact back from whichever home the row names. +async fn fetch(inline: Option, key: Option) -> error::Result> { + match (inline, key) { + (Some(inline), _) => Ok(Some(inline)), + (None, Some(key)) => get_object(&key).await.map(Some), + (None, None) => Ok(None), + } +} + +/// Drop the objects nothing points at any more. Best-effort: an object left +/// behind costs storage, and there is nothing useful to do about it in the path +/// of a run that has already finished. +async fn forget_objects(keys: &[Option; 2]) { + for key in keys.iter().flatten() { + delete_object(key).await; + } +} + +/// Whether the artifact was stored. `false` means this instance has no object +/// storage to put it in. +/// +/// The INSTANCE store, where every other internal worker artifact lives — bun +/// bundles, python wheels, job logs, the global cache. Not the workspace's: +/// that bucket is the one workspace members read and write through +/// `job_helpers/*` and `wmill.write_s3_file`, so a manifest there is one any +/// member could replace, and the next deferring run would hand dbt an +/// attacker-chosen `defer_relation` for every unbuilt `ref()` while holding the +/// script's warehouse credentials. Its compiled SQL would be readable there too, +/// for a project the reader may have no access to. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn put_object(key: &str, value: String) -> error::Result { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return Ok(false); + }; + store + .put(&ObjectPath::from(key), bytes::Bytes::from(value).into()) + .await + .map_err(|e| Error::internal_err(format!("storing the dbt state at {key}: {e:#}")))?; + Ok(true) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn get_object(key: &str) -> error::Result { + let Some(store) = windmill_object_store::get_object_store().await else { + return Err(missing_storage()); + }; + let bytes = windmill_object_store::attempt_fetch_bytes(store, key).await?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::internal_err(format!("the stored dbt state is not valid UTF-8: {e}"))) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn delete_object(key: &str) { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return; + }; + if let Err(e) = store.delete(&ObjectPath::from(key)).await { + tracing::warn!("dbt: could not drop the superseded state object {key}: {e:#}"); + } +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn delete_object(_key: &str) {} + +/// A build without the instance store carries no client at all, so an oversized +/// artifact has nowhere but the row, and a row naming a key was written by a +/// worker that did have one. +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn put_object(_key: &str, _value: String) -> error::Result { + Ok(false) +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn get_object(_key: &str) -> error::Result { + Err(missing_storage()) +} + +fn missing_storage() -> Error { + Error::BadRequest( + "the dbt state for this environment is in the instance's object storage, which this \ + worker cannot reach: it is no longer configured, or this worker was built without \ + object-storage support" + .to_string(), + ) +} + +/// The stored state this run resolves its unbuilt `ref()`s through, materialised +/// into the job directory at `STATE_DIR`. +#[derive(Clone, Debug)] +pub(crate) struct Deferral { + /// The run that published the state, so the job log and the result can say + /// what this one deferred to. + pub published_by: Uuid, + /// Whether the state carries `run_results.json` beside its manifest. A build + /// recovered by node retry publishes without one, and that is the only file a + /// `result:` selector reads. + pub has_run_results: bool, +} + +/// Materialise the environment's state so `--state` has a directory to read. +/// +/// Refused rather than run without deferral where nothing is published: the run +/// would build against a `ref()` resolving into the schema it writes, and fail +/// deep inside dbt with a relation-not-found the caller has no way to connect +/// back to a missing state. +pub(crate) async fn prepare_deferral( + p: &PreparedProject, + w_id: &str, + job_dir: &str, + conn: &Connection, +) -> error::Result { + if p.script_path.is_empty() { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the state a previous run of this script \ + published, so it needs a deployed script; a preview run has no environment to have \ + published one" + .to_string(), + )); + } + // An environment is the warehouse, the target and where they RESOLVE to, and + // a `profiles.yml` that templates its schema or database resolves somewhere + // this runtime does not render. Two renderings would then share one + // environment, and a deferral after the value changed would resolve every + // unbuilt `ref()` through the previous location's manifest. + if p.templated_location { + return Err(Error::BadRequest( + "this project's profile selects its schema or database with a template, which dbt \ + renders and Windmill does not — so two environments cannot be told apart and a \ + deferral could resolve through the wrong one's manifest. Spell the target's schema \ + and database literally to use `defer`" + .to_string(), + )); + } + let Some(state) = load(p, w_id, conn).await? else { + return Err(Error::BadRequest(format!( + "no dbt state is stored for this environment ({}), so a `ref()` this run does not \ + build has no relation to resolve to. It is published by a successful run that adds \ + nothing of its own: one overriding `select` or `vars` does not publish, and neither \ + does any run of a descriptor that interpolates a `{{{{ }}}}` placeholder into `vars` \ + or a `$var:` into `env` — those describe a model set the caller's arguments decided. \ + Run this script once without `defer` and without overrides", + environment_label(p) + ))); + }; + let has_run_results = state.run_results.is_some(); + write_state_dir( + &PathBuf::from(job_dir).join(STATE_DIR), + state.run_results.as_deref(), + StateManifest::Bytes(state.manifest), + ) + .await?; + Ok(Deferral { published_by: state.job_id, has_run_results }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Every component earns its place: a deferral resolves relation NAMES, so + // state published where those names meant something else has to read as no + // state at all rather than as state that silently no longer fits. + #[test] + fn a_moved_profile_is_another_environment() { + let here = environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")); + assert_eq!( + here, + environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("other"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("dev"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("prod"), Some("marts"), Some("wh")) + ); + assert_ne!( + here, + environment_key( + Some("main"), + Some("prod"), + Some("analytics"), + Some("other_db") + ) + ); + } + + // A target name and a schema are both the user's own strings, so a component + // carrying the separator must not be able to spell another tuple's key: a + // profile moving between the two would read as the same environment and + // defer through the manifest of relations that are somewhere else. + #[test] + fn a_component_cannot_spell_another_environments_key() { + assert_ne!( + environment_key(Some("main"), Some("prod|analytics"), Some("scratch"), None), + environment_key(Some("main"), Some("prod"), Some("analytics|scratch"), None) + ); + assert_ne!( + environment_key(Some("main"), Some("prod"), Some("a"), Some("b|c")), + environment_key(Some("main"), Some("prod"), Some("a|b"), Some("c")) + ); + // A component the profile leaves out is the same environment as one it + // spells empty: there is no target named "". + assert_eq!( + environment_key(Some("main"), None, Some("a"), None), + environment_key(Some("main"), Some(""), Some("a"), Some("")) + ); + } + + // Two environments must not queue behind one advisory lock, which is what a + // digest folded through a signed parse did for every one whose top bit is + // set — half of them. + #[test] + fn each_environment_gets_its_own_publication_lock() { + let mut seen = std::collections::HashSet::new(); + for i in 0..64 { + seen.insert(publication_lock( + "ws", + "f/a/p", + &format!("main|prod|s{i}|db"), + )); + } + assert_eq!(seen.len(), 64); + assert_eq!( + publication_lock("ws", "f/a/p", "e"), + publication_lock("ws", "f/a/p", "e") + ); + } +} diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 54457f81b0..b2e69d26d6 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re // the user's own ATTACH. `// data_test` lines append verifier probes that run // against the freshly-materialized target and raise (failing the run) on // violation. Returns `None` when there is no materialize annotation or the -// target isn't a ducklake (only ducklake is materialized in v1). +// target isn't a ducklake: only ducklake has a write engine, and a `dbt://` +// target is recorded by the generic job path instead (worker.rs, +// `record_declared_warehouse_write`). fn build_materialized_query( query: &str, partition_value: Option<&str>, diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 5e9666722a..3f6205f100 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -147,14 +147,17 @@ pub fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { Ok(()) } -/// Two-tier cache load: check local disk first, then fall back to instance object store. +/// Two-tier cache load: check local disk first, then fall back to the shared object store. +/// +/// "Shared" is the worker group's own store when its config overrides one, the instance store +/// otherwise — see [`windmill_object_store::get_cache_object_store`]. /// Returns `(hit, log_message)`. pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let started = std::time::Instant::now(); if let Ok(mut x) = windmill_object_store::attempt_fetch_bytes(os, _remote_path).await { @@ -200,13 +203,15 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bo } } -/// Whether this worker can push to the instance object store at all — the features are +/// Whether this worker can push to the shared object store at all — the features are /// compiled in and a store is loaded. False on builds without them, where `save_cache` /// only ever writes to the worker's own disk. pub async fn object_store_available() -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] { - windmill_object_store::get_object_store().await.is_some() + windmill_object_store::get_cache_object_store() + .await + .is_some() } #[cfg(not(all(feature = "enterprise", feature = "parquet")))] { @@ -214,14 +219,14 @@ pub async fn object_store_available() -> bool { } } -/// Whether a binary/bundle is in the instance object store, ignoring the local cache. +/// Whether a binary/bundle is in the shared object store, ignoring the local cache. /// /// The deploy-time prebuild asks this rather than [`exists_in_cache`]: a copy on the /// building worker's own disk is exactly the state the prebuild exists to fix, so /// answering from it would latch a failed upload into a permanent skip. pub async fn exists_in_object_store(_remote_path: &str) -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { return os .head(&windmill_object_store::object_store_reexports::Path::from( _remote_path, @@ -241,18 +246,18 @@ pub async fn ensure_pushed_to_object_store(remote_path: &str) -> error::Result<( return Ok(()); } Err(error::Error::ExecutionErr(format!( - "the binary was built but did not reach the instance object store at {remote_path}, \ + "the binary was built but did not reach the object store at {remote_path}, \ so no other worker can load it" ))) } -/// Check whether a binary/bundle exists in local cache or instance object store. +/// Check whether a binary/bundle exists in local cache or the shared object store. pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { if tokio::fs::metadata(&bin_path).await.is_ok() { return true; } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { return os .get(&windmill_object_store::object_store_reexports::Path::from( _remote_path, @@ -264,7 +269,7 @@ pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { } } -/// Two-tier cache write: upload to instance object store, then copy to local disk. +/// Two-tier cache write: upload to the shared object store, then copy to local disk. pub async fn save_cache( local_cache_path: &str, _remote_cache_path: &str, @@ -275,7 +280,7 @@ pub async fn save_cache( let mut _cached_to_s3 = false; #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { use windmill_object_store::object_store_reexports::Path; let file_to_cache = if is_dir { let tar_path = format!( diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index e3e7868548..6059936af6 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -33,9 +33,11 @@ pub mod common; mod config; mod csharp_executor; +mod dbt_column_index; mod dbt_engine; mod dbt_executor; mod dbt_profiles; +mod dbt_state; #[cfg(feature = "private")] mod dedicated_worker_ee; mod dedicated_worker_oss; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 1ede0933cb..03328b82cb 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -14,7 +14,7 @@ use windmill_common::{ }; use windmill_queue::MiniPulledJob; -use windmill_parser::Typ; +use windmill_parser::{MainArgSignature, Typ}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -40,6 +40,43 @@ lazy_static::lazy_static! { const COMPOSER_LOCK_SPLIT: &str = "\nLOCK\n"; +static PHP_PARSER_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + +pub(crate) async fn parse_php_signature( + code: &str, + main_override: Option, +) -> Result { + parse_php_signature_with_slot(code, main_override, &PHP_PARSER_SLOT).await +} + +async fn parse_php_signature_with_slot( + code: &str, + main_override: Option, + slot: &'static tokio::sync::Semaphore, +) -> Result { + let acquire = slot.acquire(); + tokio::pin!(acquire); + // Retain the acquisition across the warning to preserve its FIFO queue position. + let permit = tokio::select! { + permit = &mut acquire => permit, + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => { + tracing::warn!("Waiting over a second for PHP signature parser capacity"); + acquire.await + } + } + .map_err(to_anyhow)?; + let code = code.to_owned(); + tokio::task::spawn_blocking(move || { + // Parsing walks the entire AST. Keep its stack off async workers and retain + // the process-wide CPU limit even if the awaiting job is cancelled. + let _permit = permit; + windmill_parser_php::parse_php_signature(&code, main_override) + }) + .await + .map_err(|e| error::Error::internal_err(format!("PHP signature parsing task failed: {e}")))? + .map_err(Into::into) +} + pub fn parse_php_imports(code: &str) -> anyhow::Result> { let find_requirements = code .lines() @@ -333,11 +370,9 @@ pub async fn handle_php_job( let main_override = job.script_entrypoint_override.as_deref(); let write_wrapper_f = async { - let args = windmill_parser_php::parse_php_signature( - inner_content, - main_override.map(ToString::to_string), - )? - .args; + let args = parse_php_signature(inner_content, main_override.map(ToString::to_string)) + .await? + .args; let args_to_include = args .iter() @@ -491,3 +526,51 @@ try {{ .await?; read_result(job_dir, None).await } + +#[cfg(test)] +mod tests { + use super::parse_php_signature_with_slot; + + #[test] + fn cancelled_parse_retains_slot_until_blocking_work_finishes() { + static SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + let runtime = tokio::runtime::Builder::new_current_thread() + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let blocker = runtime.spawn_blocking(move || { + started_tx.send(()).unwrap(); + let _ = release_rx.recv(); + }); + + runtime.block_on(async { + started_rx.await.unwrap(); + let parse = tokio::spawn(parse_php_signature_with_slot( + ") { use crate::global_cache::build_tar_and_push; - use windmill_object_store::get_object_store; + use windmill_object_store::get_cache_object_store; while let Some(task) = rx.recv().await { - if let Some(os) = get_object_store().await { + if let Some(os) = get_cache_object_store().await { match build_tar_and_push(os, task.venv_path.clone(), task.cache_dir, None, false).await { Ok(()) => { @@ -117,6 +118,12 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); +/// Every file exchanged with a job is UTF-8 by construction, so the interpreter +/// must agree. A job env carries no locale: Linux then picks UTF-8 on its own +/// (PEP 540), Windows picks the ANSI code page. Applied after the whitelisted +/// envs so the protocol is not the user's to opt out of. +pub const PYTHON_UTF8_ENVS: [(&str, &str); 1] = [("PYTHONUTF8", "1")]; + /// Render loader.py with the TEMP_SCRIPT_REFS placeholder substituted by a /// Python dict literal. Preview jobs pass a path -> temp-hash map so relative /// imports resolve from not-yet-deployed local content; deployed runs pass @@ -138,7 +145,7 @@ pub fn has_relative_imports(content: &str) -> bool { use crate::global_cache::pull_from_tar; #[cfg(all(feature = "enterprise", feature = "parquet"))] -use windmill_object_store::OBJECT_STORE_SETTINGS; +use windmill_object_store::get_cache_object_store; use crate::{ common::{ @@ -221,9 +228,9 @@ fn filter_pip_local_dependencies(lines: Vec) -> (Vec, Vec, compiled_deps: &[Regex]) -> (Vec, Vec) { - let (ignored, kept): (Vec, Vec) = lines - .into_iter() - .partition(|s| !s.starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s))); + let (ignored, kept): (Vec, Vec) = lines.into_iter().partition(|s| { + !s.trim_start().starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s)) + }); (kept, ignored) } @@ -317,6 +324,11 @@ pub async fn uv_pip_compile( "compile", "-q", "--no-header", + // The `#`-line filter applied to the output below only catches whole-line + // annotations, and uv's annotation style is configurable: `[pip] + // annotation-style = "line"` in the worker HOME's uv.toml emits them inline + // ("anyio==4.15.1 # via httpx"), which that filter keeps. + "--no-annotate", file, "--strip-extras", "-o", @@ -818,7 +830,7 @@ pub async fn handle_python_job( del pre_args[k] kwargs = inner_script.preprocessor(**pre_args) kwrags_json = res_to_json(kwargs, type(kwargs)) - with open("args.json", 'w') as f: + with open("args.json", 'w', encoding="utf-8") as f: f.write(kwrags_json)"# ) } else { @@ -853,7 +865,7 @@ pub async fn handle_python_job( _pre_result = asyncio.run(_pre_result) kwargs = _pre_result if _pre_result is not None else {{}} _pre_json = json.dumps(kwargs, separators=(',', ':'), default=str) - with open("args.json", 'w') as f: + with open("args.json", 'w', encoding="utf-8") as f: f.write(_pre_json) sys.stdout.write("wm_res[preprocessed_args]:" + _pre_json + "\n") sys.stdout.flush()"# @@ -874,11 +886,11 @@ import sys from {module_dir_dot} import {last} as inner_script from wmill.client import _run_workflow -with open("args.json") as f: +with open("args.json", encoding="utf-8") as f: kwargs = json.load(f, strict=False) {transforms} -with open("checkpoint.json") as f: +with open("checkpoint.json", encoding="utf-8") as f: checkpoint = json.load(f, strict=False) result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") @@ -904,12 +916,12 @@ try: print("") print("--- WAC: complete ---") output_json = json.dumps(output, separators=(',', ':'), default=str) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: f.write(output_json) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} extra = e.__dict__ if extra and len(extra) > 0: @@ -936,7 +948,7 @@ import sys from {module_dir_dot} import {last} as inner_script import re -with open("args.json") as f: +with open("args.json", encoding="utf-8") as f: kwargs = json.load(f, strict=False) args = {{}} {transforms} @@ -972,12 +984,12 @@ try: print("WM_STREAM: " + chunk.replace('\n', '\\n')) res = None res_json = res_to_json(res, typ) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: f.write(res_json) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} extra = e.__dict__ if extra and len(extra) > 0: @@ -1117,6 +1129,7 @@ mount {{ ) .await?, ) + .envs(PYTHON_UTF8_ENVS) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1152,6 +1165,7 @@ mount {{ ) .await?, ) + .envs(PYTHON_UTF8_ENVS) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1223,6 +1237,7 @@ mount {{ result, job, conn, + canceled_by, modules, new_args.as_ref(), )) @@ -2440,7 +2455,7 @@ pub async fn handle_python_reqs( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if OBJECT_STORE_SETTINGS.read().await.is_none() { + if get_cache_object_store().await.is_none() { (s3_pull, s3_push) = (false, false); } @@ -2511,11 +2526,23 @@ pub async fn handle_python_reqs( // Find out if there is already cached dependencies // If so, skip them let mut in_cache = vec![]; + if requirements + .iter() + .any(|r| lockfile_line_has_continuation(r)) + { + tracing::warn!(workspace_id = %w_id, job_id = %job_id, "lockfile continues entries across lines; the continued lines are dropped"); + append_logs( + job_id, + w_id, + "\n[!] lockfile continues entries across lines and the continued lines are dropped: `--hash=` pins, extras and markers written that way do not apply\n".to_string(), + conn, + ) + .await; + } for req in &requirements { - // Ignore python version annotation backed into lockfile - if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { + let Some(req) = requirement_from_lockfile_line(req) else { continue; - } + }; let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( @@ -2898,7 +2925,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet"))] if is_not_pro { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(Error::from(anyhow::anyhow!("S3 pull was canceled"))), @@ -3430,7 +3457,10 @@ pub async fn start_worker( ) .await; - let mut proc_envs = HashMap::new(); + let mut proc_envs: HashMap = PYTHON_UTF8_ENVS + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); let additional_python_paths_folders = additional_python_paths.iter().join(":"); proc_envs.insert("PYTHONPATH".to_string(), additional_python_paths_folders); proc_envs.insert("PATH".to_string(), PATH_ENV.to_string()); diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 647d569cbe..21b83b32ff 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -816,8 +816,19 @@ pub async fn handle_receive_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] #[derive(serde::Deserialize)] struct GitSyncCheck { - check_run_id: i64, - repo_url: String, + /// Absent when the repository's host has no check surface (GitLab): the + /// result then reaches the pull request through the managed comment alone. + #[serde(default)] + check_run_id: Option, + /// Only markers written before the repository URL moved out of job args + /// carry one; the resource path on the job is what is used now. + #[serde(default)] + repo_url: Option, + /// Host and path of the repository the check was created on, with no + /// credential in it. The resource path is mutable, so this is what proves + /// the resource still points where the check lives. + #[serde(default)] + repo: Option, #[serde(default)] pr_number: Option, #[serde(default)] @@ -972,19 +983,21 @@ mod git_sync_check_tests { } } -/// When an auto-pull job (carrying `__git_sync_auto_pull`) fails, roll the +/// When an auto-pull job (carrying `__git_sync_auto_pull`) completes: on success, +/// record the commit as a head the workspace reflects; on failure, roll the /// optimistic `last_synced_sha` advance back to the pre-pull value so the commit /// is retried instead of being silently treated as synced, and record the failure. +/// The recorded commit is the one the pull script reports having checked out +/// (`{sha, branch}` in its result): the branch can move between the observation +/// the marker holds and the clone. A result without it falls back to the marker. #[cfg(all(feature = "enterprise", feature = "private"))] async fn maybe_reconcile_git_sync_auto_pull( db: &DB, job_id: &uuid::Uuid, workspace_id: &str, success: bool, + result: &str, ) { - if success { - return; // the optimistic synced state is already correct - } let marker: Option = match sqlx::query_scalar!( "SELECT args->'__git_sync_auto_pull' FROM v2_job WHERE id = $1", job_id @@ -1004,12 +1017,50 @@ async fn maybe_reconcile_git_sync_auto_pull( #[derive(serde::Deserialize)] struct AutoPullMarker { repo_resource_path: String, + branch: Option, + head_sha: Option, #[serde(default)] prev_synced: std::collections::HashMap, } let Ok(m) = serde_json::from_value::(marker) else { return; }; + if success { + // The optimistic synced state is already correct; record that the workspace + // now reflects the commit, which the PR CI-test check waits for. + #[derive(serde::Deserialize)] + struct PullResult { + sha: Option, + branch: Option, + } + let applied = serde_json::from_str::(result).ok(); + let branch = applied + .as_ref() + .and_then(|r| r.branch.as_deref()) + .or(m.branch.as_deref()); + let sha = applied + .as_ref() + .and_then(|r| r.sha.as_deref()) + .or(m.head_sha.as_deref()); + if let (Some(branch), Some(sha)) = (branch, sha) { + if let Err(e) = windmill_git_sync::record_synced_head( + db, + workspace_id, + &m.repo_resource_path, + branch, + sha, + "pull", + Some(*job_id), + ) + .await + { + tracing::warn!( + "git auto-pull: failed to record synced head {sha} on {branch}: {e:#}" + ); + } + } + return; + } windmill_git_sync::record_auto_pull_failure( db, workspace_id, @@ -1098,6 +1149,70 @@ fn git_sync_push_result_pushed(result: &str) -> Option { .as_bool() } +/// When a git-sync push job pushed a commit, record it as a head the workspace +/// reflects, the way a successful pull records the commit it applied. The PR +/// CI-test check waits for that record. Best-effort: failures are logged, never +/// propagated. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maybe_record_git_sync_pushed_head( + db: &DB, + job_id: &uuid::Uuid, + workspace_id: &str, + result: &str, +) { + #[derive(serde::Deserialize)] + struct PushResult { + pushed: bool, + sha: Option, + branch: Option, + #[serde(default)] + rebased: bool, + } + let Ok(PushResult { pushed: true, sha: Some(sha), branch: Some(branch), rebased }) = + serde_json::from_str::(result) + else { + return; + }; + // A push that had to rebase sits on commits this workspace has not pulled, so the + // pushed head is not something it reflects yet; the pull those commits trigger + // records the head once they are in. + if rebased { + tracing::info!( + "git sync push: {sha} on {branch} was rebased onto unpulled commits; not recording it as synced for {workspace_id}" + ); + return; + } + let repo_path = match sqlx::query_scalar!( + "SELECT args->>'repo_url_resource_path' FROM v2_job WHERE id = $1", + job_id + ) + .fetch_optional(db) + .await + { + Ok(Some(Some(p))) => p, + Ok(_) => return, + Err(e) => { + tracing::error!("git sync push: failed to read job args: {e:#}"); + return; + } + }; + if let Err(e) = windmill_git_sync::record_synced_head( + db, + workspace_id, + &repo_path, + &branch, + &sha, + "push", + Some(*job_id), + ) + .await + { + tracing::warn!( + "git sync push: failed to record pushed head {sha} on {branch} for {workspace_id}/{repo_path}: {e:#}" + ); + } +} + /// When a git-sync push job carrying `__git_sync_open_pr` succeeds, open (or /// reopen) the PR for the branch it pushed: `wm-fork//` for a fork /// deploy, `wm_deploy/**` for a promotion deploy. Runs outbound with the @@ -1162,18 +1277,18 @@ async fn maybe_open_git_sync_deploy_pr( }; // Base = the tracked branch (resource branch, else the repo default). Also - // acts as the app-backed gate: PR creation needs the installation token. - let base = match windmill_common::git_sync_ee::get_app_repo_head_for_autopull( + // acts as the gate: PR creation needs a credential the server itself holds. + let base = match windmill_common::git_sync_ee::managed_pr_base_branch( db, workspace_id, &repo_path, ) .await { - Ok(Some((branch, _))) => branch, + Ok(Some(branch)) => branch, Ok(None) => { tracing::warn!( - "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but is not GitHub-App-backed; skipping (connect the repo through the GitHub App, or use the open-pr-on-commit workflow)" + "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but the server holds no credential for it; skipping (connect the repo through the GitHub App or a GitLab token, or use the open-pr-on-commit workflow)" ); return; } @@ -1373,22 +1488,59 @@ async fn maybe_post_git_sync_check( (None, Some(deploy)) => (true, deploy), (None, None) => return, }; - let Ok(mut check) = serde_json::from_value::(marker) else { + let Ok(check) = serde_json::from_value::(marker) else { return; }; - // Markers carry the literal resource URL (job args are persisted, so a - // `$var:`-resolved URL must not land there); interpolate before calling - // GitHub. - check.repo_url = - match windmill_common::variables::get_variable_or_self(check.repo_url, db, workspace_id) - .await - { - Ok(u) => u, - Err(e) => { - tracing::error!("git sync-check: cannot interpolate repo url: {e:#}"); - return; - } - }; + // Job args are persisted, so the repository URL is not among them: it is + // re-resolved here from the resource path the pull job carries. A marker + // written before that change still has the URL, and is honoured until the + // last such job has drained. + // The resource path is mutable, so it is only trusted when the marker also + // carries the identity to check it against. A marker written before that + // identity existed keeps using the URL it captured at enqueue, which cannot + // have been repointed since. + let repo_url = match ( + check.repo.is_some(), + row.repo_path.as_deref(), + check.repo_url.clone(), + ) { + // The resource path is mutable, so following it is only safe when the + // marker also carries the identity to check the result against. + (true, Some(path), _) => { + windmill_common::git_sync_ee::resolve_repo_url_interpolated(db, workspace_id, path) + .await + } + // A marker written before that identity existed captured the URL itself, + // which cannot have been repointed since. + (_, _, Some(url)) => { + windmill_common::variables::get_variable_or_self(url, db, workspace_id).await + } + // Neither: nothing here can prove which repository this check belongs to, + // and resolving the path anyway is how a preview reaches the wrong one. + // Leaving the check unfinished is the safe failure. + _ => { + tracing::error!( + "git sync-check: the marker carries neither a repository identity nor a url; not acting on it" + ); + return; + } + }; + let repo_url = match repo_url { + Ok(u) => u, + Err(e) => { + tracing::error!("git sync-check: cannot resolve repo url: {e:#}"); + return; + } + }; + // A resource repointed while the diff was running would otherwise close a + // check, or post a preview, on a repository that has nothing to do with it. + if check.repo.is_some() && windmill_common::git_sync_ee::repo_identity(&repo_url) != check.repo + { + tracing::warn!( + "git sync-check: the repository moved since the check was created; leaving it alone" + ); + return; + } // "In sync" on a PR that visibly changes files reads as a bug when those // files are outside the repo's sync filters — say what the scope is. let scope_note = if !is_deploy && success { @@ -1451,7 +1603,7 @@ async fn maybe_post_git_sync_check( ( "failure", "Merge conflicts with the base branch".to_string(), - "This PR cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." + "This branch cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." .to_string(), ) } else if pr_check_error.as_deref() == Some("PR_HEAD_REF_UNAVAILABLE") { @@ -1461,7 +1613,7 @@ async fn maybe_post_git_sync_check( ( "neutral", "Could not compute the deploy diff".to_string(), - "Windmill could not fetch this PR's head or enough history from GitHub to compute its merge with the base. Push again to re-run this check." + "Windmill could not fetch this branch's head, or enough history, to compute its merge with the base. Push again to re-run this check." .to_string(), ) } else if pr_check_error.is_some() { @@ -1484,20 +1636,20 @@ async fn maybe_post_git_sync_check( "success", "In sync".to_string(), format!( - "Merging this PR would make no changes to the workspace.{}", + "Merging this branch would make no changes to the workspace.{}", scope_note.as_deref().unwrap_or_default() ), ), Some((changes, settings_changed)) => { let mut lines = vec![format!( - "Merging this PR would apply {} change(s) to the workspace:\n", + "Merging this branch would apply {} change(s) to the workspace:\n", changes.len() )]; lines.extend(format_change_list(&changes)); if settings_changed { lines.push(match check.wmill_yaml_changed { - Some(true) => "\nThis PR changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), - Some(false) => "\nIndependent of this PR, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), + Some(true) => "\nThis branch changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), + Some(false) => "\nIndependent of this branch, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), None => "\nA pull also updates the workspace's git-sync settings to match the repo's wmill.yaml.".to_string(), }); } @@ -1520,19 +1672,21 @@ async fn maybe_post_git_sync_check( Some(url) => format!("{summary}\n\n[See the job in Windmill]({url})"), None => summary.clone(), }; - if let Err(e) = windmill_common::git_sync_ee::update_check_run( - db, - workspace_id, - &check.repo_url, - check.check_run_id, - conclusion, - &title, - &check_summary, - job_url.as_deref(), - ) - .await - { - tracing::error!("git sync-check: failed to update check run: {e:#}"); + if let Some(check_run_id) = check.check_run_id { + if let Err(e) = windmill_common::git_sync_ee::update_check_run( + db, + workspace_id, + &repo_url, + check_run_id, + conclusion, + &title, + &check_summary, + job_url.as_deref(), + ) + .await + { + tracing::error!("git sync-check: failed to update check run: {e:#}"); + } } // Phase 4 also maintains ONE managed comment on the PR (Cloudflare @@ -1556,7 +1710,7 @@ async fn maybe_post_git_sync_check( if let Err(e) = windmill_common::git_sync_ee::upsert_pr_comment( db, workspace_id, - &check.repo_url, + &repo_url, pr_number, marker, &body, @@ -1657,8 +1811,26 @@ pub async fn process_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { maybe_post_git_sync_check(db, &job_id, &workspace_id, true, result.get()).await; + maybe_reconcile_git_sync_auto_pull(db, &job_id, &workspace_id, true, result.get()) + .await; + maybe_record_git_sync_pushed_head(db, &job_id, &workspace_id, result.get()).await; maybe_open_git_sync_deploy_pr(db, &job_id, &workspace_id, result.get()).await; } + // A CI test job just finished: advance any open "Windmill CI tests" PR check for + // its workspace. Detached, since concluding a check calls GitHub and this loop + // completes jobs serially; the evaluation is idempotent and the poller retries. + #[cfg(all(feature = "enterprise", feature = "private"))] + if job + .trigger_kind + .as_ref() + .is_some_and(|k| k.is(windmill_common::jobs::JobTriggerKind::CiTest)) + { + let db = db.clone(); + let w_id = workspace_id.clone(); + tokio::spawn(async move { + windmill_git_sync::evaluate_and_conclude_ci_test_checks(&db, &w_id).await + }); + } // Asset-trigger fan-out: best-effort, never propagates errors. // Internal eligibility checks gate to top-level Script/Preview runs; @@ -1769,7 +1941,20 @@ pub async fn process_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { maybe_post_git_sync_check(db, &job.id, &job.workspace_id, false, result.get()).await; - maybe_reconcile_git_sync_auto_pull(db, &job.id, &job.workspace_id, false).await; + maybe_reconcile_git_sync_auto_pull(db, &job.id, &job.workspace_id, false, "").await; + } + // A failed CI test job also settles its check; same detached advance as on success. + #[cfg(all(feature = "enterprise", feature = "private"))] + if job + .trigger_kind + .as_ref() + .is_some_and(|k| k.is(windmill_common::jobs::JobTriggerKind::CiTest)) + { + let db = db.clone(); + let w_id = job.workspace_id.clone(); + tokio::spawn(async move { + windmill_git_sync::evaluate_and_conclude_ci_test_checks(&db, &w_id).await + }); } if job.is_flow_step() { if let Some(parent_job) = job.parent_job { diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs index f36a5e6fc9..1126c94d17 100644 --- a/backend/windmill-worker/src/universal_pkg_installer.rs +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -333,7 +333,7 @@ pub async fn par_install_language_dependencies_all_at_once< mark_success(path.clone(), job_id, w_id).await; #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let language_name = _language_name.to_owned(); tokio::spawn(async move { if let Err(e) = crate::global_cache::build_tar_and_push( @@ -790,7 +790,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] let s3_pull_future = if is_not_pro { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { Some(crate::global_cache::pull_from_tar( os, dep.path.clone(), @@ -893,7 +893,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let language_name = _language_name.to_string(); let platform_agnostic = _platform_agnostic; let path = dep.path.clone(); @@ -954,8 +954,7 @@ async fn print_success( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if windmill_object_store::OBJECT_STORE_SETTINGS - .read() + if windmill_object_store::get_cache_object_store() .await .is_none() { diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 3e9b7bbc9b..798cd347b1 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -1,11 +1,13 @@ use serde::Deserialize; use serde_json::value::RawValue; use serde_json::Value; +use sqlx::{Postgres, Transaction}; use uuid::Uuid; use windmill_common::error::{self, Error}; use windmill_common::scripts::ScriptLang; use windmill_common::DB; +use windmill_queue::CanceledBy; // Checkpoint model + persistence primitives live in windmill-common so the // API server can use them without pulling in the full worker crate. Re-export @@ -44,6 +46,10 @@ pub enum WacOutput { form: Option, #[serde(default)] self_approval_disabled: Option, + #[serde(default)] + skin: Option, + #[serde(default)] + description: Option, }, /// Server-side sleep — suspend the workflow for a duration without holding a worker. #[serde(rename = "sleep")] @@ -85,6 +91,122 @@ fn default_dispatch_type() -> String { "inline".to_string() } +/// What `suspend_wac_parent` did with the parent's queue row. +#[derive(Debug)] +pub enum WacPark { + /// Parked. Carries the segment that just ended, in milliseconds, for `end_wac_segment`. + Parked(Option), + /// A cancel reached the row while this segment was running, so the park was skipped. + /// Carries who cancelled, for the completion that must happen instead. + Cancelled(CanceledBy), +} + +/// Park a WAC v2 parent in the queue until `suspend` reaches 0 or `suspend_secs` +/// elapses, whichever comes first. `running` stays true so the normal pull query +/// skips the row; only the suspended pull query takes it back. The `id`/`workspace_id` +/// pair is a consistency check, not an authorization one — callers must already hold +/// the job (every one of them passes a job its own worker pulled). +/// +/// `started_at` is cleared because the parent holds no worker while parked. The pull +/// re-stamps it (`started_at = coalesce(started_at, now())`), and every path that +/// completes a job without a worker-measured duration — a cancel, the child-failure +/// handler — falls back to `now() - started_at`. Left pointing at the first segment, +/// that fallback reports the whole sleep or approval wait as execution time. +pub async fn suspend_wac_parent( + tx: &mut Transaction<'_, Postgres>, + job_id: &Uuid, + w_id: &str, + suspend: i32, + suspend_secs: f64, +) -> error::Result { + // `FOR UPDATE` orders this against a concurrent soft cancel, which writes `suspend = 0` + // and leaves acting on `canceled_by` to the next pull. Parking on top of that keeps the + // row unpullable until `suspend_until` — up to the full `sleep()` — so a cancel already + // on the row has to stand the park down rather than be overwritten by it. + let prev = sqlx::query!( + "SELECT canceled_by, canceled_reason, + (extract(epoch FROM now() - started_at) * 1000)::bigint AS segment_ms + FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 FOR UPDATE", + job_id, + w_id, + ) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to read WAC parent job {job_id}: {e}")))? + // Silently parking nothing is unrecoverable on the dispatch arm: the children are + // pushed right after and decrement a `suspend` that was never set, so the parent + // sits out its whole suspend window instead of resuming. + .ok_or_else(|| { + Error::internal_err(format!( + "WAC parent job {job_id} not in the queue of workspace {w_id} to suspend" + )) + })?; + + if let Some(username) = prev.canceled_by { + return Ok(WacPark::Cancelled(CanceledBy { + username: Some(username), + reason: prev.canceled_reason, + })); + } + + sqlx::query!( + "UPDATE v2_job_queue + SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null + WHERE id = $1 AND workspace_id = $2", + job_id, + w_id, + suspend, + suspend_secs, + ) + .execute(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to suspend WAC parent job {job_id}: {e}")))?; + + Ok(WacPark::Parked(prev.segment_ms)) +} + +/// Turn a cancel that landed mid-segment into the error the executor returns, so the job +/// completes on this pass instead of parking. Setting the worker's `canceled_by` is what +/// makes it land as `canceled` rather than `failure`: the row was cancelled after this +/// worker pulled the job, so the in-memory copy still reads as uncancelled. +/// +/// The completion charges the segment that just ended, so callers must not also hand it to +/// `end_wac_segment`. +pub(crate) fn wac_cancelled_mid_segment( + cancel: CanceledBy, + canceled_by: &mut Option, +) -> Error { + let payload = windmill_common::worker::to_raw_value(&windmill_queue::canceled_result( + cancel.reason.as_deref(), + cancel.username.as_deref(), + )); + *canceled_by = Some(cancel); + Error::ExecutionRawError(payload) +} + +/// Charge the execution segment a WAC parent just finished. Segments are metered as they +/// end rather than summed at completion, so a workflow that sleeps for days is billed for +/// the compute it used, when it used it — and the final segment is charged by the ordinary +/// completion path. +/// +/// Call this only where the parent really parks. On a rollback that goes on to complete +/// the job, the completion charges the same segment and it would be billed twice. +pub(crate) fn end_wac_segment( + _conn: &windmill_common::worker::Connection, + _job: &windmill_queue::MiniPulledJob, + _segment_ms: Option, +) { + #[cfg(feature = "cloud")] + if let (windmill_common::worker::Connection::Sql(db), Some(segment_ms)) = (_conn, _segment_ms) { + windmill_queue::meter_execution_seconds( + db, + &_job.workspace_id, + &_job.permissioned_as_email, + segment_ms, + ); + } +} + /// Parse the WAC result from result.json content. pub fn parse_wac_output(result: &RawValue) -> error::Result { serde_json::from_str(result.get()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7e620439b0..d893ec071b 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3867,7 +3867,8 @@ pub async fn run_worker( let job_result = windmill_common::log_context::with_log_context( log_ctx, async { - let result = handle_queued_job( + // Keep large job-phase futures boxed to limit debug polling frames. + let result = Box::pin(handle_queued_job( arc_job.clone(), raw_code, raw_lock, @@ -3888,7 +3889,7 @@ pub async fn run_worker( flow_runners, #[cfg(feature = "benchmark")] &mut bench, - ) + )) .await; record_job_span_status(&result); result @@ -5159,7 +5160,7 @@ async fn try_validate_schema( code, language, job.script_entrypoint_override.clone(), - )? { + ).await? { Ok(Some(schema_validator_from_main_arg_sig(&sig))) } else { Err(anyhow!("Job was expected to validate the arguments schema, but no schema was provided and couldn't be inferred from the script for language `{language:?}`. Try removing schema validation for this job").into()) @@ -5527,7 +5528,7 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); - run_language_executor( + let result = Box::pin(run_language_executor( job, conn, client, @@ -5552,8 +5553,111 @@ async fn handle_code_execution_job( &modules, false, in_pipeline, - ) - .await + )) + .await; + record_declared_warehouse_write(job, conn, code, &result).await; + result +} + +/// Record the outcome of a `// materialize manual dbt:////` +/// declaration, the way the DuckDB executor records a DuckLake target. +/// +/// Nothing generates warehouse DDL, so the script issues its own write and this +/// is the only thing that turns it into a `materialized_partition` row — the +/// relation's last writer on the run page and the graph. Language-agnostic on +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. +/// +/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` +/// rows, not from this one, so a lost row costs the relation its last writer and +/// nothing else. It must not fail a job whose write already landed. +/// +/// Shares the reach of every other runtime pipeline annotation, which is this +/// function's caller: a job handed to a dedicated worker or a flow runner never +/// passes through it, so — exactly as `// partitioned` is not resolved there — +/// such a run performs its write and records no row. +async fn record_declared_warehouse_write( + job: &MiniPulledJob, + conn: &Connection, + code: &str, + result: &error::Result>, +) { + use windmill_common::materialization::{ + MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED, + }; + // A DEPLOYED script only. The annotation is a deploy-time contract — `manual`, + // a three-segment relation, a configured warehouse — checked in + // `create_script_internal`, which also required write access to the path. A + // preview, hub or inline-flow body reaches this function without any of that, + // so honouring it there would let `jobs:run` alone restamp any relation's last + // writer from a script that never touched it. + if job.kind != JobKind::Script { + return; + } + // Cheap guard: the annotation scan is skipped for the overwhelming majority + // of jobs, which carry no `materialize` line at all. + if !code.contains("materialize") { + return; + } + let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code) + .materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt) + else { + return; + }; + // The slice this run wrote, resolved once upstream (`resolve_partition_for_job`) + // and carried in the args the cascade reads too, so a partitioned producer + // records the same identity everything else propagates. + let partition = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or_else(|| UNPARTITIONED.to_string()); + let (status, error) = match result { + Ok(_) => (MaterializationStatus::Materialized, None), + Err(e) => (MaterializationStatus::Failed, Some(e.to_string())), + }; + let recorded = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + &job.workspace_id, + windmill_common::assets::AssetKind::Dbt, + &m.target_path, + &partition, + status, + None, + None, + Some(job.id), + error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http( + client, + &job.workspace_id, + &RecordMaterializationRequest { + asset_kind: windmill_common::assets::AssetKind::Dbt, + asset_path: m.target_path.clone(), + partition, + status, + snapshot_id: None, + row_count: None, + job_id: Some(job.id), + error, + schema: None, + }, + ) + .await + } + }; + if let Err(e) = recorded { + tracing::warn!( + "recording the materialization of dbt://{} failed: {e:#}", + m.target_path + ); + } } /// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot @@ -6309,7 +6413,8 @@ mount {{ | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Nativets - | ScriptLang::Go => "//", + | ScriptLang::Go + | ScriptLang::Php => "//", _ => "", }; let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); @@ -6362,7 +6467,7 @@ mount {{ .await; if let Connection::Sql(db) = conn { - volume_setup = crate::volume_oss::setup_volumes_sql_worker( + volume_setup = Box::pin(crate::volume_oss::setup_volumes_sql_worker( &volume_mounts, db, &job.workspace_id, @@ -6375,10 +6480,10 @@ mount {{ language, &mut envs, &mut shared_mount, - ) + )) .await?; } else if let Connection::Http(http) = conn { - volume_setup = crate::volume_oss::setup_volumes_http_worker( + volume_setup = Box::pin(crate::volume_oss::setup_volumes_http_worker( &volume_mounts, http, &job.workspace_id, @@ -6391,7 +6496,7 @@ mount {{ language, &mut envs, &mut shared_mount, - ) + )) .await?; } } @@ -6887,7 +6992,7 @@ mount {{ if let Some(ref vol_client) = volume_setup.client { if let Connection::Sql(db) = conn { - crate::volume_oss::sync_volumes_sql_worker( + Box::pin(crate::volume_oss::sync_volumes_sql_worker( &volume_setup.states, &volume_setup.writable, vol_client, @@ -6897,13 +7002,13 @@ mount {{ worker_name, conn, result.is_ok(), - ) + )) .await; } } if let Connection::Http(http) = conn { - crate::volume_oss::sync_volumes_http_worker( + Box::pin(crate::volume_oss::sync_volumes_http_worker( &volume_setup.states, &volume_setup.writable, http, @@ -6912,7 +7017,7 @@ mount {{ worker_name, conn, result.is_ok(), - ) + )) .await; } @@ -6947,7 +7052,7 @@ mount {{ result } -pub fn parse_sig_of_lang( +pub async fn parse_sig_of_lang( code: &str, language: Option<&ScriptLang>, main_override: Option, @@ -6982,10 +7087,9 @@ pub fn parse_sig_of_lang( ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?), ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?), #[cfg(feature = "php")] - ScriptLang::Php => Some(windmill_parser_php::parse_php_signature( - code, - main_override, - )?), + ScriptLang::Php => { + Some(crate::php_executor::parse_php_signature(code, main_override).await?) + } #[cfg(not(feature = "php"))] ScriptLang::Php => None, #[cfg(feature = "rust")] diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 34f46307cf..31dddcaf5f 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -4438,6 +4438,8 @@ async fn push_next_flow_job( .as_deref() .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) { + // A step with its own on-behalf-of carries a cached dispatch address, up to one + // notify poll stale; accepted, see `get_email_from_permissioned_as`. let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; check_tag_available_for_workspace_internal( db, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 57b859075a..1f3b9f1d56 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -19,14 +19,17 @@ use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::jobs::JobKind; use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; -use windmill_common::scripts::ScriptHash; +use windmill_common::scripts::{ + deploy_relocked_version, fetch_script_for_update, hash_script, ScriptHash, ScriptModule, +}; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, WorkspaceDependenciesPrefetched, }; -use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; +use windmill_dep_map::trigger_dependents::trigger_dependents_to_recompute_dependencies; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; @@ -38,8 +41,10 @@ use windmill_common::{ scripts::ScriptLang, DB, }; +use windmill_dep_map::lock_hash::record_lock_hashes; pub use windmill_dep_map::{ extract_referenced_paths, extract_relative_imports, process_relative_imports, + refresh_dependency_map, }; use windmill_git_sync::{ handle_deployment_metadata, tally_deployed_object_changes, DeployedObject, @@ -86,10 +91,13 @@ use crate::{ /// has the toolchain and, since the cache key is per OS/arch, the platform the runtime /// workers use. Deploys that supply their own lock never reach a dependency job at all and /// queue theirs from `create_script_internal` instead. -async fn maybe_queue_binary_prebuild(db: &DB, job: &MiniPulledJob, lock: &str) -> Result<()> { - let (Some(hash), Some(path), Some(lang)) = - (job.runnable_id, job.runnable_path.clone(), job.script_lang) - else { +async fn maybe_queue_binary_prebuild( + db: &DB, + job: &MiniPulledJob, + hash: ScriptHash, + lock: &str, +) -> Result<()> { + let (Some(path), Some(lang)) = (job.runnable_path.clone(), job.script_lang) else { return Ok(()); }; let Some(prebuild) = @@ -166,7 +174,7 @@ async fn handle_build_binary_job( if !crate::global_cache::object_store_available().await { return Ok(to_raw_value_owned(json!({ "status": "skipped", - "reason": "this worker cannot reach the instance object store, so the binary \ + "reason": "this worker cannot reach an object store, so the binary \ would not be shared with other workers", }))); } @@ -283,6 +291,13 @@ pub async fn handle_dependency_job( job.runnable_path() ); let script_path = job.runnable_path(); + let w_id = &job.workspace_id; + + let triggered_by_relative_import = job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default(); // A build pass reads the same script data but writes none of the deploy state below, // including the `lock_error_logs` stamp on a fetch failure: the version it builds is @@ -296,14 +311,48 @@ pub async fn handle_dependency_job( *deployment_tallied = true; } + // A relative-import relock deploys nothing until `commit_relock` says so, while the + // caller's fallback tally assumes a failed dependency job left a deployed version behind. + // Claim the tally here; the failure path hands it back once it has minted the version + // that carries the error. + if triggered_by_relative_import { + *deployment_tallied = true; + } + + // A relative-import relock locks the path's live version as of now, not the hash captured + // when the job was pushed: a deploy can land during the debounce delay, after which that + // hash names an archived version. What it generates is committed against the live version + // re-read under a row lock, so a deploy landing mid-generation is caught there too. + let target_hash = if triggered_by_relative_import { + Some(ScriptHash(live_head_hash(db, w_id, script_path).await?)) + } else { + job.runnable_id + }; + // `JobKind::Dependencies` job store either: // - A saved script `hash` in the `script_hash` column. // - Preview raw lock and code in the `queue` or `job` table. - let script_data = &match job.runnable_id { + let script_data = &match target_hash { + // Read straight from the database: the cache pins a version's data under its hash for + // as long as this worker lives, and the live version may still be waiting on its own + // dependency job's lock, which lands in place. A run resolving to it on this worker + // would then get no lock from the cache at all. + Some(hash) if triggered_by_relative_import => { + let raw = cache::script::fetch_script_from_db(db, hash, std::panic::Location::caller()) + .await?; + Cow::Owned(std::sync::Arc::new(cache::ScriptData { + lock: raw.lock, + code: raw.content, + modules: raw.modules, + })) + } Some(hash) => match cache::script::fetch(&Connection::from(db.clone()), hash).await { Ok(d) => Cow::Owned(d.0), Err(e) => { - if !is_build_job { + // The live version of a relative-import relock is what runs resolve to, and + // `lock_error_logs` on it takes it out of resolution; the job carries the + // error instead, since it deployed nothing. + if !is_build_job && !triggered_by_relative_import { let logs2 = sqlx::query_scalar!( "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2", &job.id, @@ -348,12 +397,6 @@ pub async fn handle_dependency_job( .await; } - let triggered_by_relative_import = job - .args - .as_ref() - .map(|x| x.get("triggered_by_relative_import").is_some()) - .unwrap_or_default(); - // Extract temp_script_refs from job args (path -> hash mapping for temp storage) let temp_script_refs: Option> = job .args @@ -391,20 +434,17 @@ pub async fn handle_dependency_job( ) .await; + let (deployment_message, parent_path) = + get_deployment_msg_and_parent_path_from_args(job.args.clone()); + match content { Ok(content) => { - if job.runnable_id.is_none() { + let Some(current_hash) = target_hash else { // it a one-off raw script dependency job, no need to update the db return Ok(to_raw_value_owned( json!({ "status": "Successful lock file generation", "lock": content }), )); - } - - let current_hash = job.runnable_id.unwrap_or(ScriptHash(0)); - let w_id = &job.workspace_id; - - let (deployment_message, parent_path) = - get_deployment_msg_and_parent_path_from_args(job.args.clone()); + }; // Generate lockfiles for module files (if any). // @@ -441,7 +481,9 @@ pub async fn handle_dependency_job( occupancy_metrics, &raw_workspace_dependencies_o, module.lock.as_deref(), - triggered_by_relative_import, + // A module that was never locked has nothing a skip could hand + // back; the path's lock is the parent script's, not its own. + triggered_by_relative_import && module.lock.is_some(), script_path, None, "script", @@ -464,34 +506,83 @@ pub async fn handle_dependency_job( None }; - // We do not create new row for this update - // That means we can keep current hash and just update lock - // Also store lockfile hash for dependency change detection - let lockfile_hash = windmill_common::scripts::hash_script(&content); - let updated_modules_json = updated_modules - .as_ref() - .and_then(|m| serde_json::to_value(m).ok()); - sqlx::query!( - "WITH update_lock AS ( - UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + let deployed_hash = if triggered_by_relative_import { + match commit_relock( + db, + w_id, + script_path, + current_hash, + Some(&content), + updated_modules.as_ref(), + None, + deployment_message.clone(), ) - INSERT INTO lock_hash (workspace_id, path, lockfile_hash) - VALUES ($3, $4, $5) - ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - &content, - ¤t_hash.0, - w_id, - script_path, - &lockfile_hash, - updated_modules_json - ) - .execute(db) - .await?; + .await? + { + RelockOutcome::Deployed(hash) => hash, + RelockOutcome::Unchanged => { + let log_msg = "\nLock unchanged: no new version deployed"; + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + // The imports may have moved even though the result did not, and the + // map is what this importer's next skip check reads. + refresh_dependency_map( + db, + w_id, + script_path, + &parent_path, + &script_data.code, + &job.script_lang, + ) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock unchanged, no new version deployed", "lock": content }), + )); + } + RelockOutcome::Superseded(head) => { + let log_msg = format!( + "\nVersion {head} was deployed while this lock was generated; discarding it and queueing a relock of that version" + ); + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock generation superseded by a newer version", "lock": content }), + )); + } + } + } else { + // We do not create new row for this update + // That means we can keep current hash and just update lock + // Also store lockfile hash for dependency change detection + let lockfile_hash = windmill_common::scripts::hash_script(&content); + let updated_modules_json = updated_modules + .as_ref() + .and_then(|m| serde_json::to_value(m).ok()); + sqlx::query!( + "WITH update_lock AS ( + UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + ) + INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + VALUES ($3, $4, $5) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + &content, + ¤t_hash.0, + w_id, + script_path, + &lockfile_hash, + updated_modules_json + ) + .execute(db) + .await?; - // `lock` has been updated; invalidate the cache. - // Since only worker that ran this Dependency Job has the cache - // we do not need to think about invalidating cache for other workers. - cache::script::invalidate(current_hash); + // `lock` has been updated; invalidate the cache. + // Since only worker that ran this Dependency Job has the cache + // we do not need to think about invalidating cache for other workers. + cache::script::invalidate(current_hash); + current_hash + }; // The version only became runnable now, so this process still resolves the path to // the one before it. Only the runnable-hash cache: the import-side caches ignore the // lock, so evicting this process' half of that pair here would key a bundle by a @@ -504,7 +595,7 @@ pub async fn handle_dependency_job( &db, &w_id, DeployedObject::Script { - hash: current_hash, + hash: deployed_hash, path: script_path.to_string(), parent_path: parent_path.clone(), }, @@ -565,7 +656,7 @@ pub async fn handle_dependency_job( }); } - if let Err(e) = maybe_queue_binary_prebuild(db, job, &content).await { + if let Err(e) = maybe_queue_binary_prebuild(db, job, deployed_hash, &content).await { tracing::error!(%e, "error queueing the auto-build binary job for {script_path}"); } @@ -583,14 +674,49 @@ pub async fn handle_dependency_job( .await? .flatten() .unwrap_or_else(|| "no logs".to_string()); - sqlx::query!( - "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", - &format!("{logs2}\n{error}"), - &job.runnable_id.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; + let error_logs = format!("{logs2}\n{error}"); + if let (true, Some(hash)) = (triggered_by_relative_import, target_hash) { + // The same shape a failed deploy leaves: a version without a lock that carries + // the error, so it shows on the script while runs keep resolving to the last + // version that has one. Only that version is the caller's fallback to tally; + // one that landed meanwhile owns its own lock, and a commit that failed left + // nothing. + match commit_relock( + db, + w_id, + script_path, + hash, + None, + None, + Some(&error_logs), + deployment_message.clone(), + ) + .await + { + Ok(RelockOutcome::Deployed(_)) => *deployment_tallied = false, + Ok(RelockOutcome::Superseded(_)) => { + if let Err(e) = + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await + { + tracing::error!(%e, "error queueing a relock of {script_path}") + } + } + Ok(RelockOutcome::Unchanged) => {} + Err(e) => { + tracing::error!(%e, "error recording the failed relock of {script_path}") + } + } + } else { + sqlx::query!( + "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", + &error_logs, + &job.runnable_id.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + } Err(Error::ExecutionErr(format!( "Error locking file: {error}\n\nlogs:\n{}", remove_ansi_codes(&logs2) @@ -598,6 +724,136 @@ pub async fn handle_dependency_job( } } } + +/// The version of `script_path` that runs resolve to, which is what a relative-import relock +/// locks. `NotFound` when the path holds none, which a job pushed for a path since archived or +/// deleted reports as its own failure. +async fn live_head_hash(db: &DB, w_id: &str, script_path: &str) -> error::Result { + sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND archived = false ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + )) + }) +} + +enum RelockOutcome { + /// A new version carrying the result is the live one. + Deployed(ScriptHash), + /// The live version already holds this lock and these module locks; nothing was written. + Unchanged, + /// The live version is no longer the one the lock was generated for; nothing was written. + Superseded(ScriptHash), +} + +/// Commits what a relative-import relock produced against the path's live version, read under +/// a row lock so relocks of one path serialize on it. +/// +/// A result equal to the live version's lock and module locks writes nothing: the importer's +/// dependencies did not move, and a new version would deploy byte-identical content and then +/// walk its own importers for nothing. A `lock` of `None` is a failed generation and always +/// deploys, as the version that carries the error. +async fn commit_relock( + db: &DB, + w_id: &str, + script_path: &str, + generated_for: ScriptHash, + lock: Option<&str>, + modules: Option<&HashMap>, + lock_error_logs: Option<&str>, + deployment_message: Option, +) -> error::Result { + let mut tx = db.begin().await?; + let mut head = None; + for _ in 0..4 { + head = fetch_script_for_update(script_path, w_id, &mut *tx).await?; + if head.is_some() { + break; + } + // Having waited on the live version's row lock, the statement re-checked that row + // once the holder committed, found it archived, and returned nothing: the successor + // the holder inserted is not in the statement's snapshot. A fresh statement sees it, + // unless yet another writer got there first, so this goes around a few times before + // concluding the path holds no live version. + } + let Some(head) = head else { + return Err(Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + ))); + }; + if head.hash != generated_for { + // A deploy landed while the lock was generated. It carried its own lock or queued its + // own dependency job, and this lock describes content that is no longer live. + return Ok(RelockOutcome::Superseded(head.hash)); + } + let lock_hash_entry = lock.map(|lock| (script_path.to_string(), hash_script(lock))); + if let Some(lock) = lock { + let modules_unchanged = modules.map_or(true, |m| head.modules.as_ref() == Some(m)); + if head.lock.as_deref() == Some(lock) && modules_unchanged { + // The hash row is still written, and under the same row lock: a version deployed + // before lock hashes were recorded has none, so its importers cannot skip until + // it does, and a deploy that takes the lock next must not have the hash it records + // overwritten by this one. + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + return Ok(RelockOutcome::Unchanged); + } + } + let new_hash = deploy_relocked_version( + &mut tx, + head, + deployment_message, + lock, + modules, + lock_error_logs, + ) + .await?; + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + Ok(RelockOutcome::Deployed(ScriptHash(new_hash))) +} + +/// Queues another relative-import relock of `script_path`, through the same push the fan-out +/// uses. The version live now was deployed while a lock was generated for its predecessor; +/// when that deploy was a sibling relock it queued nothing for this path, and the result just +/// discarded may have been the one generated against the current imports. +async fn requeue_relock( + db: &DB, + job: &MiniPulledJob, + script_path: &str, + deployment_message: Option, + parent_path: Option, +) -> error::Result<()> { + let already_visited = job + .args + .as_ref() + .and_then(|x| x.get("already_visited")) + .and_then(|v| serde_json::from_str::>(v.get()).ok()) + .unwrap_or_default(); + trigger_dependents_to_recompute_dependencies( + &job.workspace_id, + vec![DependencyDependent { + importer_path: script_path.to_string(), + importer_kind: "script".to_string(), + importer_node_ids: None, + }], + deployment_message, + parent_path, + &job.permissioned_as_email, + &job.created_by, + &job.permissioned_as, + db, + already_visited, + ) + .await +} + fn remove_ansi_codes(s: &str) -> String { lazy_static::lazy_static! { static ref ANSI_REGEX: regex::Regex = regex::Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap(); @@ -2892,8 +3148,12 @@ async fn try_skip_relock( } // Fetch existing lock based on runnable type - let lock = match runnable_type { - "script" => sqlx::query_scalar!( + let lock = match (runnable_type, existing_lock) { + // A script's module asks with the script's own type and hands over the lock it last + // deployed with. The path's lock below is the parent script's, and a module given + // that loses whatever it resolves on its own. + ("script", Some(module_lock)) => Some(module_lock.to_string()), + ("script", None) => sqlx::query_scalar!( "SELECT lock FROM script WHERE path = $1 AND workspace_id = $2 AND lock IS NOT NULL AND deleted = false ORDER BY created_at DESC LIMIT 1", base_path, @@ -2903,7 +3163,7 @@ async fn try_skip_relock( .await? .flatten(), - "flow" | "app" => existing_lock.map(|s| s.to_string()), + ("flow" | "app", existing_lock) => existing_lock.map(|s| s.to_string()), _ => None, }; diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 6a165901e4..4ef8c1e90b 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.803.0"; +export const VERSION = "v1.811.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/TESTING.md b/cli/TESTING.md index 235a95c4b1..df0311677d 100644 --- a/cli/TESTING.md +++ b/cli/TESTING.md @@ -22,6 +22,12 @@ Pure local tests — no backend, no database. Uses `bunfig.unit.toml` (no preloa Examples: `git_unit`, `lint_command_unit`, `tar_creation_unit`, `workspace_conflicts_unit` +`mock.module()` mocks the module for the whole `bun test` process, not for the file +that installs it, and `mock.restore()` does not undo it. A file that mocks a module +must hand back its real exports in `afterAll` (see +`schedule_push_permissioned_as_unit`), or it silently rewires whichever file runs +next — and the run order is the directory's, so it differs between Linux and Windows. + ### Integration tests Require a running backend and PostgreSQL. The `setup.ts` preload builds the backend @@ -29,6 +35,28 @@ binary and starts a shared backend instance. Examples: `sync_pull_push`, `dev_server`, `standalone_commands` +## Module mocks + +`mock.module` replaces a module for the **whole process**, and it does reach modules that +were already imported — a stub one file installs lands on a consumer an earlier file +loaded. + +Handing the module back in `afterAll` is not a reliable undo. Files do run one at a time +(a root-level `afterAll` completes before the next file's body evaluates), so it looks +like it should be — but stubbing `bundle.ts` and restoring it that way still left +`raw_app_svelte_plugin_unit.test.ts` asserting against an empty bundle, green on Linux +and red on Windows, where the `readdir` file order differs. Treat a stub as permanent for +the run. + +So the rule is about what you stub, not how you clean up: **stub only a module no other +in-process suite imports.** Check with `grep -rl "" test/` before reaching +for one. A suite that drives the CLI through a spawned process is out of reach of a +module mock and doesn't count. + +`raw_app_push_policy_unit.test.ts` is the worked example: it stubs `gen/services.gen.ts`, +which passes the rule because nothing else in `test/` imports the three API functions it +replaces, and deliberately does not stub `bundle.ts`, which failed it. + ## AI Benchmark Caveats The repo-level benchmark CLI lives under `ai_evals/`, but it currently depends on diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 7772dafedd..0d4ad634c8 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + getWmillYamlPath, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; import lintCommand from "./lint.ts"; @@ -21,9 +25,11 @@ import newCommand from "./new.ts"; import generateAgentsCommand from "./generate_agents.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; export interface AppFile { + guests?: boolean; value: any; public?: boolean; summary: string; @@ -110,6 +116,50 @@ export function replaceInlineScripts( export function isExecutionModeAnonymous(app: any) { return app?.["policy"]?.["execution_mode"] == "anonymous"; } +export function isExecutionModeGuest(app: any) { + return app?.["policy"]?.["execution_mode"] == "guest"; +} +export type AppExecutionMode = "anonymous" | "guest" | "publisher" | "viewer"; +/** The access mode is the one policy field a tracked app keeps, as `public` (anonymous) + * or `guests` (guest); the rest of the policy is preserved from the deployed app on + * push (see `generatingPolicy`). */ +export function markAccessFromPolicy(app: any) { + if (isExecutionModeAnonymous(app)) { + app.public = true; + } else if (isExecutionModeGuest(app)) { + app.guests = true; + } +} +/** The mode the tracked file states, or `undefined` when it states none — the + * normal case, since a pull writes only the two open-access markers. `viewer` + * and `publisher` have no marker of their own, so a file can only name them + * through a policy block it was hand-written with. */ +function statedExecutionMode(app: any): AppExecutionMode | undefined { + if (app?.["public"] ?? isExecutionModeAnonymous(app)) { + return "anonymous"; + } + if (app?.["guests"] ?? isExecutionModeGuest(app)) { + return "guest"; + } + const mode = app?.["policy"]?.["execution_mode"]; + return mode === "viewer" || mode === "publisher" ? mode : undefined; +} + +/** The mode this push deploys under. A file that states one is authoritative, in + * both directions. Otherwise the two open-access markers are all it says, so + * their absence closes a deployed `anonymous`/`guest` app back down to + * `publisher` — while a deployed `viewer` is not a grant those markers revoke, + * so it carries over rather than widening to `publisher`. */ +export function executionModeForPush( + localApp: any, + deployedPolicy: Policy | undefined, +): AppExecutionMode { + const stated = statedExecutionMode(localApp); + if (stated) { + return stated; + } + return deployedPolicy?.execution_mode === "viewer" ? "viewer" : "publisher"; +} export async function pushApp( workspace: string, remotePath: string, @@ -133,16 +183,11 @@ export async function pushApp( //ignore } - let remoteOnBehalfOf: string | undefined; - let remoteOnBehalfOfEmail: string | undefined; - if (app?.policy) { - remoteOnBehalfOf = app.policy.on_behalf_of; - remoteOnBehalfOfEmail = app.policy.on_behalf_of_email; - } + // `app.policy` is cleared a few lines down, so capture it first: it is the + // base the regenerated policy is built on. + const deployedPolicy: Policy | undefined = app?.policy; - if (isExecutionModeAnonymous(app)) { - app.public = true; - } + markAccessFromPolicy(app); // console.log(app); if (app) { app.policy = undefined; @@ -155,26 +200,19 @@ export async function pushApp( const localApp = (await yamlParseFile(path)) as AppFile; replaceInlineScripts(localApp.value, localPath, true); + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext + ); await generatingPolicy( localApp, remotePath, - localApp?.["public"] ?? - localApp?.["policy"]?.["execution_mode"] == "anonymous" + executionModeForPush(localApp, deployedPolicy), + basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of) ); - const preserveFields: { preserve_on_behalf_of?: boolean } = {}; - if (permissionedAsContext?.userIsAdminOrDeployer) { - if (app) { - if (localApp.policy && remoteOnBehalfOf) { - (localApp.policy as any).on_behalf_of = remoteOnBehalfOf; - (localApp.policy as any).on_behalf_of_email = remoteOnBehalfOfEmail; - preserveFields.preserve_on_behalf_of = true; - log.info(`Preserving ${remoteOnBehalfOfEmail ?? remoteOnBehalfOf} as permissioned_as for app ${remotePath}`); - } - } - // On create: backend applies folder defaults - } - // extra_perms goes through /acls/* — strip from the body so a perms-only // edit never bumps the app version (see applyExtraPermsDiff for details). const { extra_perms: localPerms, ...localAppBody } = localApp as AppFile & { @@ -230,18 +268,76 @@ export async function pushApp( export async function generatingPolicy( app: any, path: string, - publicApp: boolean + executionMode: AppExecutionMode, + base: Policy | undefined ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updatePolicy(app.value, undefined); - app.policy.execution_mode = publicApp ? "anonymous" : "publisher"; + app.policy = await windmillUtils.updatePolicy(app.value, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; } } +/** What the regenerated policy starts from: the deployed one, so a push keeps + * settings the tracked file doesn't record; on a first push, whatever the file + * states. The run identity rides along only when `claimsOnBehalfOf` — never + * from the file, never from a pusher who may not preserve one, since `wmill` + * is regularly pointed at servers older than the rewrite that would fix it. */ +export function basePolicy( + localApp: any, + deployedPolicy: Policy | undefined, + claimsOnBehalfOf: boolean +): Policy | undefined { + const stated = deployedPolicy ?? (localApp?.policy as Policy | undefined); + if (!stated || claimsOnBehalfOf) { + return stated; + } + const base: Policy = { ...stated }; + delete base.on_behalf_of; + delete base.on_behalf_of_email; + return base; +} + +/** Claim the run-as identity the regenerated policy carries over from the + * deployed app. Only a deployed identity may be claimed, never one the tracked + * file states — a repo doesn't get to pick who an app runs as. Without the flag + * the backend rewrites `on_behalf_of` to whoever is pushing, and it only honors + * the flag for an admin or a `wm_deployers` member, so a caller who is neither + * doesn't get to claim it here either. */ +export function preserveOnBehalfOfFields( + remotePath: string, + deployedPolicy: Policy | undefined, + permissionedAsContext: PermissionedAsContext | undefined +): { preserve_on_behalf_of?: boolean } { + const onBehalfOf = deployedPolicy?.on_behalf_of; + if (!permissionedAsContext?.userIsAdminOrDeployer || !onBehalfOf) { + return {}; + } + log.info( + `Preserving ${deployedPolicy?.on_behalf_of_email ?? onBehalfOf} as permissioned_as for app ${remotePath}` + ); + return { preserve_on_behalf_of: true }; +} + +/** The policy is written wholesale by the deploy, so the fields it does not + * derive from the tracked sources have to survive the trip. The policy builder + * has already recomputed what it can — the triggerables on both paths, plus the + * S3 rules on the low-code one, which `updateRawAppPolicy` has no equivalent of + * and so carries over. This sets the two left: the access mode, and the legacy + * `triggerables`, which still grant execution (the backend folds them into + * `triggerables_v2` at run time) and so are dropped rather than carried, or a + * deployed app would keep being able to run runnables this push removed. */ +export function finalizeDerivedPolicy( + policy: Policy, + executionMode: AppExecutionMode +) { + policy.triggerables = undefined; + policy.execution_mode = executionMode; +} + async function list(opts: GlobalOptions & { includeDraftOnly?: boolean; json?: boolean }) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -410,10 +506,23 @@ async function push( absoluteFilePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } else { - await pushApp(workspace.workspaceId, remotePath, absoluteFilePath); + await pushApp( + workspace.workspaceId, + remotePath, + absoluteFilePath, + undefined, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), + ); log.info(colors.bold.underline.green("App pushed")); } } diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 9f83d0322a..8d82edb9df 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -1,6 +1,9 @@ import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; @@ -15,9 +18,24 @@ import { readdir } from "node:fs/promises"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { deepEqual, readTextFile } from "../../utils/utils.ts"; -import { replaceInlineScripts, repopulateFields } from "./app.ts"; +import { + type AppExecutionMode, + basePolicy, + executionModeForPush, + finalizeDerivedPolicy, + markAccessFromPolicy, + preserveOnBehalfOfFields, + replaceInlineScripts, + repopulateFields, +} from "./app.ts"; +import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER, RECORDINGS_FOLDER } from "./app_metadata.ts"; +import { + NEVER_DEPLOYED_DIRS, + NEVER_DEPLOYED_FILES, +} from "../../utils/app_files.ts"; import { writeIfChanged } from "../../utils/utils.ts"; import { yamlOptions } from "../sync/sync.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; @@ -27,6 +45,7 @@ import { } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; export interface AppFile { + guests?: boolean; runnables?: any; custom_path?: string; public?: boolean; @@ -309,13 +328,11 @@ async function collectAppFiles( const relativePath = basePath + entry.name; if (entry.isDirectory()) { - // Skip the runnables, node_modules, and sql_to_apply subfolders + // The backend folder deploys as `value.runnables`, not as a bundled + // file; the rest reach the server through no channel at all. if ( entry.name === APP_BACKEND_FOLDER || - entry.name === "node_modules" || - entry.name === "dist" || - entry.name === ".claude" || - entry.name === "sql_to_apply" + NEVER_DEPLOYED_DIRS.has(entry.name) ) { continue; } @@ -327,13 +344,11 @@ async function collectAppFiles( } await readDirRecursive(fullPath + SEP, relativePath + "/"); } else if (entry.isFile()) { - // Skip generated/metadata files that shouldn't be part of the app + // `raw_app.yaml` deploys as the request's metadata rather than as a + // bundled file; the rest reach the server through no channel at all. if ( entry.name === "raw_app.yaml" || - entry.name === "package-lock.json" || - entry.name === "DATATABLES.md" || - entry.name === "AGENTS.md" || - entry.name === "wmill.d.ts" + NEVER_DEPLOYED_FILES.has(entry.name) ) { continue; } @@ -353,6 +368,7 @@ export async function pushRawApp( localPath: string, message?: string, defaultTs: "bun" | "deno" = "bun", + permissionedAsContext?: PermissionedAsContext, ): Promise { if (alreadySynced.includes(localPath)) { return; @@ -369,9 +385,11 @@ export async function pushRawApp( } catch { //ignore } - if (app?.["policy"]?.["execution_mode"] == "anonymous") { - app.public = true; - } + // `app.policy` is cleared a few lines down, so capture it first. `raw_app.yaml` + // records none of the policy, so anything the deploy drawer set is only here. + const deployedPolicy: Policy | undefined = app?.policy; + + markAccessFromPolicy(app); // console.log(app); if (app) { app.policy = undefined; @@ -419,10 +437,21 @@ export async function pushRawApp( // Create a temporary app object for policy generation const appForPolicy = { ...localApp, runnables }; + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext, + ); await generatingPolicy( appForPolicy, remotePath, - localApp?.["public"] ?? false, + executionModeForPush(localApp, deployedPolicy), + basePolicy( + localApp, + deployedPolicy, + !!preserveFields.preserve_on_behalf_of, + ), ); const files = await collectAppFiles(localPath); @@ -477,6 +506,7 @@ export async function pushRawApp( path: remotePath, summary: localApp.summary, policy: appForPolicy.policy, + ...preserveFields, deployment_message: message, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, @@ -526,15 +556,13 @@ export async function pushRawApp( export async function generatingPolicy( app: any, path: string, - publicApp: boolean, + executionMode: AppExecutionMode, + base: Policy | undefined, ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updateRawAppPolicy( - app.runnables, - app.policy, - ); - app.policy.execution_mode = publicApp ? "anonymous" : "publisher"; + app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; @@ -559,6 +587,10 @@ async function pushRawAppCommand( filePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7a383fcfeb..863690ce7d 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -4,7 +4,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; -import { dirname, sep as SEP } from "node:path"; +import { dirname, sep as SEP, resolve as pathResolve } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; @@ -21,11 +21,16 @@ import { } from "../../core/context.ts"; import { resolve, track_job, pollForJobResult } from "../script/script.ts"; import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; -import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + SyncOptions, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { collectPathScriptPaths, replaceInlineScripts, @@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo if (!validatePath(remotePath)) { return; } + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); - await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); + await pushFlow( + workspace.workspaceId, + remotePath, + filePath, + opts.message, + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) + ); log.info(colors.bold.underline.green("Flow pushed")); } diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts index f6a35429ef..d031c16b3f 100644 --- a/cli/src/commands/lint/lint.ts +++ b/cli/src/commands/lint/lint.ts @@ -31,10 +31,18 @@ import { import { isFlowInlineScriptPath, isAppInlineScriptPath, - isRawAppPath, + isFolderResourcePathAnyFormat, getFolderSuffix, + getScriptBasePathFromModulePath, } from "../../utils/resource_folders.ts"; -import { exts } from "../script/script.ts"; +import { isFilesetResource } from "../../utils/utils.ts"; +import { + exts, + findContentFile, + hasScriptExt, + isModuleEntryMetadata, + UnresolvableScriptContentFileError, +} from "../script/script.ts"; interface LintOptions extends GlobalOptions { json?: boolean; @@ -67,6 +75,9 @@ export interface LintReport { const YAML_FILE_REGEX = /\.ya?ml$/i; const NATIVE_TRIGGER_REGEX = /\.[^.]+_native_trigger\.ya?ml$/i; +// The metadata suffixes `findContentFile` resolves a flat script from. `.yml` is +// deliberately absent, since the push does not accept it there either. +const FLAT_SCRIPT_METADATA_REGEX = /\.script\.(yaml|json|lock)$/; function normalizePath(p: string): string { return p.replaceAll(SEP, "/"); @@ -643,6 +654,83 @@ export async function checkMissingLocks( return issues; } +/** + * Whether a path is a script's own metadata, as opposed to metadata the push + * deploys through some parent: a folder resource's inline scripts, a fileset's + * children (arbitrarily named, so one may be spelled exactly like a script's + * metadata) and the files of a module or dbt bundle all belong to that parent. + * + * Takes the path as the SYNC ROOT spells it, like the push. Relative to the + * lint target the enclosing folder is gone whenever the target IS that folder; + * absolute, the classifiers match their suffixes ANYWHERE in the string, so a + * checkout under `acme.app` reads as one app and nothing is ever reported. + */ +function isStandaloneScriptMetadata(rootedPath: string): boolean { + // Both suffix formats, because the dotted/non-dotted setting is read from the + // invocation directory and an explicit lint target may not share it. + if ( + isFolderResourcePathAnyFormat(rootedPath) || + isFilesetResource(rootedPath) + ) { + return false; + } + // A module folder keeps its metadata inside itself (`__mod/script.yaml`), + // which is standalone even though every other path under `__mod/` is not. + if (isModuleEntryMetadata(rootedPath)) return true; + if (getScriptBasePathFromModulePath(rootedPath) !== undefined) return false; + return FLAT_SCRIPT_METADATA_REGEX.test(rootedPath); +} + +/** + * `findContentFile` quotes the paths it was given back in its errors, so the + * lint target's own prefix comes off them again. Anchored at a path start: a + * plain substring replace of `f/` also eats the one inside `conf/`, mangling + * the very filename the message is telling the reader to delete. + */ +function relativizeMessage(message: string, prefix: string): string { + if (!prefix) return message; + const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return message.replaceAll(new RegExp(`(^|[\\s(])${escaped}/`, "g"), "$1"); +} + +/** + * Script metadata files that cannot be paired with exactly one content file: + * the push refuses those, and no metadata format makes them deployable, so the + * inactive twin of a format switch (`foo.script.json` in a yaml repo) is dead + * weight worth reporting even though the push skips it rather than refusing it. + * + * Resolved through `findContentFile` rather than by probing `exts` directly, so + * lint and push agree on what counts as paired: a dbt project's descriptor is + * optional and its absence is not an orphan, while two content files beside one + * metadata file is just as undeployable as none. It classifies what it is given + * and looks under `syncRoot`, so where the command was invoked from is not part + * of the answer. + */ +async function checkOrphanScriptMetadata( + syncRoot: string, + prefix: string, + metadataPaths: string[], +): Promise { + const issues: FileIssue[] = []; + for (const metadataPath of metadataPaths) { + const rootedPath = prefix ? `${prefix}/${metadataPath}` : metadataPath; + try { + await findContentFile(rootedPath, syncRoot); + } catch (e) { + if (!(e instanceof UnresolvableScriptContentFileError)) { + log.debug(`Failed to resolve content file for ${rootedPath}: ${e}`); + continue; + } + issues.push({ + path: metadataPath, + target: "script", + errors: [relativizeMessage(e.message, prefix)], + }); + } + } + return issues; +} + export async function runLint( opts: LintOptions, directory?: string, @@ -674,8 +762,16 @@ export async function runLint( const root = await FSFSElement(targetDirectory, [], false); const validator = new WindmillYamlValidator(); + // Walked paths are relative to the lint target; this puts them back the way + // the sync root spells them, which is what the two below are written against. + const syncRoot = await findSyncRoot(targetDirectory); + const metadataPrefix = normalizePath( + path.relative(syncRoot, targetDirectory), + ); + const warnings: LintWarning[] = []; const issues: FileIssue[] = []; + const scriptMetadataPaths: string[] = []; let scannedFiles = 0; let validatedFiles = 0; let validFiles = 0; @@ -689,6 +785,17 @@ export async function runLint( const normalizedPath = normalizePath(entry.path); scannedFiles += 1; + + // Collected before the YAML filter below: `.script.lock` and `.script.json` + // are metadata too, and both fail the push when nothing pairs with them. + if ( + isStandaloneScriptMetadata( + metadataPrefix ? `${metadataPrefix}/${normalizedPath}` : normalizedPath, + ) + ) { + scriptMetadataPaths.push(normalizedPath); + } + if (!YAML_FILE_REGEX.test(normalizedPath)) { continue; } @@ -727,6 +834,16 @@ export async function runLint( } } + // Unconditional: unlike a missing lock, metadata with no content file fails + // every push, so there is no mode in which it is acceptable. + issues.push( + ...(await checkOrphanScriptMetadata( + syncRoot, + metadataPrefix, + scriptMetadataPaths, + )), + ); + // Check for missing locks if --locks-required is set if (opts.locksRequired) { const lockIssues = await checkMissingLocks(opts, explicitTargetDirectory); @@ -820,6 +937,15 @@ async function lint(opts: LintOptions & { watch?: boolean }, directory?: string) } } +/** + * Whether a changed file can change what a lint run reports: metadata in any of + * its formats, and the content files whose presence is what keeps that metadata + * from being an orphan. + */ +function affectsLint(filename: string): boolean { + return /\.(ya?ml|json|lock)$/i.test(filename) || hasScriptExt(filename); +} + async function lintWatch(opts: LintOptions, directory?: string) { const { watch } = await import("node:fs"); const targetDir = directory ? path.resolve(process.cwd(), directory) : process.cwd(); @@ -842,7 +968,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { let debounce: ReturnType | null = null; watch(targetDir, { recursive: true }, (_event, filename) => { - if (!filename || !filename.toString().endsWith(".yaml") && !filename.toString().endsWith(".yml")) return; + if (!filename || !affectsLint(filename.toString())) return; if (debounce) clearTimeout(debounce); debounce = setTimeout(runAndReport, 300); }); @@ -853,7 +979,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { const command = new Command() .description( - "Validate Windmill flow, schedule, and trigger YAML files in a directory", + "Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file", ) .arguments("[directory:string]") .option("--json", "Output results in JSON format") diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index 5f9e713f42..f6173a28bb 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -413,7 +413,13 @@ export function parseMuteAnnotations(content: string): { // Comment prefix for `volume:` annotations. Deliberately NOT `commentPrefix` // above (which returns `--` for SQL): volume annotations are only recognized for // the languages the backend/frontend recognize them for — mirrors -// `asset_inference.rs:comment_prefix` and `infer.ts:getCommentPrefix` (SQL → none). +// `asset_inference.rs:comment_prefix` and `infer.ts:getCommentPrefix` (SQL → none), +// minus `php`. Those two recognize PHP, but a PHP script cannot reach this map: it +// has no wasm asset parser, so `fallbackParse` handles it and that scan breaks on +// the mandatory ` { const project = basePath + "__dbt/dbt_project.yml"; - return (await stat(project).then(() => true).catch(() => false)) + const onDisk = baseDir ? path.join(baseDir, project) : project; + return (await stat(onDisk).then(() => true).catch(() => false)) ? project : undefined; } @@ -1137,7 +1141,14 @@ async function readScriptContent(filePath: string): Promise { } } -export async function findContentFile(filePath: string) { +/** + * The script file `filePath`'s metadata belongs to. `baseDir`, when given, is + * where the disk lookups happen, leaving `filePath` classified as written: the + * layout helpers below match their suffixes ANYWHERE in a path, so a caller + * that prefixed a checkout named `repo__mod` would have it read as the module. + */ +export async function findContentFile(filePath: string, baseDir?: string) { + const onDisk = (p: string) => (baseDir ? path.join(baseDir, p) : p); // Folder layout: __mod/script.yaml -> __mod/script.ts const isModuleFolderMeta = isModuleEntryMetadata(filePath); const toCandidate = (ext: string) => @@ -1161,7 +1172,7 @@ export async function findContentFile(filePath: string) { const validCandidates = ( await Promise.all( candidates.map((x) => { - return stat(x) + return stat(onDisk(x)) .catch(() => undefined) .then((x) => x?.isFile()) .then((e) => { @@ -1181,6 +1192,7 @@ export async function findContentFile(filePath: string) { const dbtCandidate = toCandidate("__dbt/" + DBT_DESCRIPTOR_NAME); const dbtProject = await collidingDbtProject( dbtCandidate.slice(0, -("__dbt/" + DBT_DESCRIPTOR_NAME).length), + baseDir, ); const nonDbtCandidates = validCandidates.filter((c) => c !== dbtCandidate); if (dbtProject && nonDbtCandidates.length > 0) { diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2702e13714..d75dc81912 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -104,7 +104,7 @@ export async function downloadZip( // from v1 the on-behalf-of address is stripped below, so the tarball sends the // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs - // on flow / script / app rows. Default-off on the server protects cross- + // on script / flow / app / variable rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities // missing in the target workspace; the CLI sync flow explicitly wants them. const baseParams = `&plain_secret=${plainSecrets ?? false @@ -150,7 +150,18 @@ export async function downloadZip( } if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + log.info( + colors.red( + `Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` + + (workspace.name !== workspace.workspaceId + ? ` (resolved from profile '${workspace.name}')` + : "") + + `.\n` + + `Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` + + ` - check 'wmill workspace list' (the 'workspace id' column)\n` + + ` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)` + ) + ); } else { log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); if (body) log.info(colors.red(body)); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index bab9967b3c..2159ea2539 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -76,7 +76,8 @@ import { } from "../../utils/utils.ts"; import { getEffectiveSettings, - getWorkspaceNames, + inferWsNameFromProfile, + resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, SyncOptions, @@ -85,7 +86,10 @@ import { WorkspaceEntryConfig, } from "../../core/conf.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { preCheckPermissionedAs } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + preCheckPermissionedAs, +} from "../../core/permissioned_as.ts"; import { fromWorkspaceSpecificPath, toWorkspaceSpecificPath, @@ -142,12 +146,13 @@ import { extractCurrentMapping, } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; -import { isExecutionModeAnonymous } from "../app/app.ts"; +import { markAccessFromPolicy } from "../app/app.ts"; import { APP_BACKEND_FOLDER, generateAppLocksInternal, RECORDINGS_FOLDER, } from "../app/app_metadata.ts"; +import { deploysWithRawApp } from "../../utils/app_files.ts"; import { isFlowPath, isAppPath, @@ -429,37 +434,6 @@ export function computeWsSpecificFlagOnlyPushes( return out; } -// Resolve workspace name from a --branch override (git branch → workspace name). -// Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch( - opts: SyncOptions, - branchName: string, -): string { - const match = findWorkspaceByGitBranch(opts.workspaces, branchName); - return match ? match[0] : branchName; -} - -// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, -// then --workspace → matching config key (incl. when --base-url is set). Returns -// undefined when no flag-based resolution applies; callers then fall back to -// inferWsNameFromProfile on the resolved workspace profile. -export function resolveWsNameForConfigFromFlags( - opts: SyncOptions & { branch?: string; workspace?: string }, -): string | undefined { - if (opts.branch) { - return resolveWsNameFromBranch(opts, opts.branch); - } - if (opts.workspace) { - // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, - // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. - const validKeys = getWorkspaceNames(opts.workspaces); - if (validKeys.includes(opts.workspace)) { - return opts.workspace; - } - } - return undefined; -} - // Warn if --workspace overrides auto-detected branch or if workspace not in config. function warnWorkspaceOverride( opts: SyncOptions, @@ -507,33 +481,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { return wsName; } -// After resolveWorkspace, infer the workspace config name from the resolved profile -// by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile( - opts: SyncOptions, - profile: { remote: string; workspaceId: string }, -): string | undefined { - if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter( - (k) => k !== "commonSpecificItems", - ); - for (const name of wsNames) { - const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; - if (!entry?.baseUrl) continue; - try { - const entryUrl = new URL(entry.baseUrl).toString(); - const profileUrl = new URL(profile.remote).toString(); - const entryWsId = entry.workspaceId ?? name; - if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { - return name; - } - } catch { - continue; - } - } - return undefined; -} - // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< T extends GlobalOptions & SyncOptions & { repository?: string }, @@ -1393,9 +1340,7 @@ export function ZipFSElement( }; } - if (isExecutionModeAnonymous(app)) { - app.public = true; - } + markAccessFromPolicy(app); app.policy = undefined; yield { isDirectory: false, @@ -1413,9 +1358,7 @@ export function ZipFSElement( log.error(`Failed to parse app.yaml at path: ${p}`); throw error; } - if (rawApp?.["policy"]?.["execution_mode"] == "anonymous") { - rawApp.public = true; - } + markAccessFromPolicy(rawApp); // console.log("rawApp", rawApp); rawApp.policy = undefined; // custom_path is derived from the file path, don't store it @@ -2076,20 +2019,18 @@ export async function elementsToMap( } if (isRawAppFile(path)) { - // FSFSElement builds paths with the platform separator, while the checks - // below are written with "/": without normalizing, none of them match on - // Windows and the push collector's own exclusions become perpetual diffs. + // FSFSElement builds paths with the platform separator, while + // `deploysWithRawApp` is written with "/": without normalizing it matches + // nothing on Windows and the push collector's own exclusions become + // perpetual diffs. const suffix = path .split(getFolderSuffix("raw_app") + SEP) .pop() ?.replaceAll(SEP, "/"); - if ( - suffix?.startsWith("dist/") || - suffix?.startsWith(RECORDINGS_FOLDER + "/") || - suffix == "wmill.d.ts" || - suffix == "package-lock.json" || - suffix == "DATATABLES.md" - ) { + // A file no push sends is not a change to track. Listing it leaves it + // pending forever — nothing ever uploads it — and pushing it redeploys + // the whole app, reassigning its run-as user, to ship nothing. + if (suffix && !deploysWithRawApp(suffix)) { continue; } } @@ -5544,27 +5485,19 @@ export async function push( return; } - let permissionedAsContext: PermissionedAsContext | undefined = undefined; - if (parseSyncBehavior(opts.syncBehavior) >= 1) { - const user = await wmill.whoami({ workspace: workspace.workspaceId }); - const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + const permissionedAsContext: PermissionedAsContext | undefined = + await buildPermissionedAsContext( + workspace.workspaceId, + opts.syncBehavior, ); - permissionedAsContext = { - userCache: new Map(), - userIsAdminOrDeployer, - userEmail: user.email, - }; - + if (permissionedAsContext) { // ws_specific_flag changes have no content payload, so they don't // affect permissioned_as resolution — filter them out before the // pre-check (which expects only added/edited/deleted). await preCheckPermissionedAs( changes.filter((c) => c.name !== "ws_specific_flag"), - user.email, - userIsAdminOrDeployer, + permissionedAsContext.userEmail, + permissionedAsContext.userIsAdminOrDeployer, opts.acceptOverridingPermissionedAsWithSelf ?? false, !!process.stdin.isTTY, ); @@ -6208,7 +6141,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // Flow folder doesn't exist locally — delete on server @@ -6253,7 +6186,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // App folder doesn't exist locally — delete on server @@ -6299,7 +6232,11 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message, defaultTs: opts.defaultTs }, + { + message: opts.message, + defaultTs: opts.defaultTs, + permissionedAsContext, + }, ); } else { // The entire raw app folder was deleted locally, diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 83b62f9eb4..4e6b0eeb16 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -23,7 +23,7 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { GlobalOptions, isSuperset, @@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; +import { readEffectiveSyncBehavior } from "../../core/conf.ts"; type Trigger = { http: HttpTrigger; @@ -222,9 +224,12 @@ export async function pushTrigger( } } +// `enabled` is operational state a sync deliberately does not carry: the server strips it from +// the workspace export and the push below never sends it, so a created trigger comes up enabled +// and pausing one stays a local decision. type NativeTriggerFile = Omit< NativeTrigger, - "external_id" | "workspace_id" | "error" + "external_id" | "workspace_id" | "error" | "enabled" >; export async function pushNativeTrigger( @@ -262,6 +267,7 @@ export async function pushNativeTrigger( service_config: result.service_config, error: result.error, summary: result.summary, + enabled: result.enabled, }; log.debug(`Native trigger ${serviceName}/${externalId} exists on remote`); } catch { @@ -620,8 +626,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise; } /** @@ -152,37 +154,42 @@ export async function pushVariable( log.debug(`Variable ${remotePath} does not exist on remote`); } + // extra_perms is synced independently via /acls/* (see applyExtraPermsDiff) + // so a perm-only edit never rewrites the variable value. Strip the field from + // the body that goes to update_variable / create_variable and treat it as a + // separate step both for the up-to-date short-circuit and after the write. + const { extra_perms: localPerms, ...localVariableBody } = localVariable; + if (variable) { - if (isSuperset(localVariable, variable)) { + if (isSuperset(localVariableBody, variable)) { log.debug(`Variable ${remotePath} is up-to-date`); - return; - } + } else { + log.debug(`Variable ${remotePath} is not up-to-date, updating`); - log.debug(`Variable ${remotePath} is not up-to-date, updating`); - - // Apply is_secret only when it differs from the remote (the value is always - // sent, so the server allows the flag change). Upgrades (non-secret->secret) - // always apply; downgrades only when explicitly allowed (single-file push) — - // see allowSecretDowngrade. `undefined` leaves the flag untouched. - let nextIsSecret: boolean | undefined = undefined; - if (localVariable.is_secret !== variable.is_secret) { - if (localVariable.is_secret) { - nextIsSecret = true; - } else if (allowSecretDowngrade) { - nextIsSecret = false; + // Apply is_secret only when it differs from the remote (the value is always + // sent, so the server allows the flag change). Upgrades (non-secret->secret) + // always apply; downgrades only when explicitly allowed (single-file push) — + // see allowSecretDowngrade. `undefined` leaves the flag untouched. + let nextIsSecret: boolean | undefined = undefined; + if (localVariableBody.is_secret !== variable.is_secret) { + if (localVariableBody.is_secret) { + nextIsSecret = true; + } else if (allowSecretDowngrade) { + nextIsSecret = false; + } } - } - await wmill.updateVariable({ - workspace, - path: remotePath.replaceAll(SEP, "/"), - alreadyEncrypted: !plainSecrets, - requestBody: { - ...localVariable, - is_secret: nextIsSecret, - ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), - }, - }); + await wmill.updateVariable({ + workspace, + path: remotePath.replaceAll(SEP, "/"), + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariableBody, + is_secret: nextIsSecret, + ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), + }, + }); + } } else { log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await wmill.createVariable({ @@ -190,11 +197,22 @@ export async function pushVariable( alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath.replaceAll(SEP, "/"), - ...localVariable, + ...localVariableBody, ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), }, }); } + + // Synced whether or not the body changed. No refetch: folder perms are never + // merged onto item.extra_perms, and the update/create body carries no + // extra_perms, so the value getVariable read above is still the remote one. + await applyExtraPermsDiff( + workspace, + "variable", + remotePath.replaceAll(SEP, "/"), + localPerms, + (variable as any)?.extra_perms, + ); } async function push( diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1fcce7a43f..0c746d2261 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null { } export const GLOBAL_CONFIG_OPT = { noCdToRoot: false }; -function findWmillYaml(): string | null { + +// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir. +function locateWmillYaml(): string | null { const startDir = resolve(process.cwd()); const isInGitRepo = isGitRepository(); const gitRoot = isInGitRepo ? getGitRepoRoot() : null; @@ -176,6 +178,13 @@ function findWmillYaml(): string | null { currentDir = parentDir; } + return foundPath; +} + +function findWmillYaml(): string | null { + const startDir = resolve(process.cwd()); + const foundPath = locateWmillYaml(); + // If wmill.yaml was found in a parent directory, warn the user and change working directory if ( !GLOBAL_CONFIG_OPT.noCdToRoot && @@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } +/** + * Look up one `workspaces` entry, for diagnostics only. readConfigFile() must + * not be used for that: it chdirs to the config's directory, exits on an + * unsupported syncBehavior and throws on a malformed file. A diagnostic may + * never fail or relocate the command it is diagnosing. + */ +export async function peekWorkspaceEntry( + workspaceName: string +): Promise { + if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) { + return undefined; + } + const wmillYamlPath = locateWmillYaml(); + if (!wmillYamlPath) { + return undefined; + } + try { + const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions; + const workspaces = + conf?.workspaces ?? + conf?.gitBranches ?? + conf?.environments ?? + conf?.git_branches; + const entry = (workspaces as any)?.[workspaceName]; + return typeof entry === "object" && entry !== null ? entry : undefined; + } catch (e) { + log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`); + return undefined; + } +} + let legacyConfigWarned = false; export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { @@ -630,6 +670,83 @@ export async function getEffectiveSettings( return effective; } +// Resolve workspace name from a --branch override (git branch → workspace name). +// Falls back to using the branch value as-is (backward compat: old key = branch name). +function resolveWsNameFromBranch(opts: SyncOptions, branchName: string): string { + const match = findWorkspaceByGitBranch(opts.workspaces, branchName); + return match ? match[0] : branchName; +} + +// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, +// then --workspace → matching config key (incl. when --base-url is set). Returns +// undefined when no flag-based resolution applies; callers then fall back to +// inferWsNameFromProfile on the resolved workspace profile. +export function resolveWsNameForConfigFromFlags( + opts: SyncOptions & { branch?: string; workspace?: string } +): string | undefined { + if (opts.branch) { + return resolveWsNameFromBranch(opts, opts.branch); + } + if (opts.workspace) { + // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, + // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. + const validKeys = getWorkspaceNames(opts.workspaces); + if (validKeys.includes(opts.workspace)) { + return opts.workspace; + } + } + return undefined; +} + +/** + * Match a workspace config entry to a resolved workspace profile by remote + + * workspace id. The fallback for when no flag names the entry outright. + */ +export function inferWsNameFromProfile( + opts: SyncOptions, + profile: { remote: string; workspaceId: string } +): string | undefined { + if (!opts.workspaces) return undefined; + for (const name of getWorkspaceNames(opts.workspaces)) { + const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; + if (!entry?.baseUrl) continue; + try { + const entryUrl = new URL(entry.baseUrl).toString(); + const profileUrl = new URL(profile.remote).toString(); + const entryWsId = entry.workspaceId ?? name; + if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { + return name; + } + } catch { + continue; + } + } + return undefined; +} + +/** + * `syncBehavior` as the workspace being pushed to sees it. The top level alone + * misses a `workspaces..overrides.syncBehavior`, which is where a repo + * that varies settings per workspace puts it, and the entry to read is the one + * `--workspace` names — falling back to the profile, then to the git branch — + * the same order `sync push` resolves it in. + */ +export async function readEffectiveSyncBehavior( + opts: { workspace?: string }, + profile?: { remote: string; workspaceId: string } +): Promise { + const config = await readConfigFile({ warnIfMissing: false }); + const named = resolveWsNameForConfigFromFlags({ ...config, ...opts }); + const effective = await getEffectiveSettings( + config, + undefined, + false, + true, + named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined) + ); + return effective.syncBehavior; +} + const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]); /** diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 7017136a66..0e6b0805d2 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.803.0"; +export const VERSION = "1.811.1"; diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 502157f0a7..80b27ea52e 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -20,6 +20,7 @@ import { import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts"; import { readConfigFile, + peekWorkspaceEntry, findWorkspaceByGitBranch, getEffectiveWorkspaceId, getWmillYamlPath, @@ -219,6 +220,9 @@ async function tryResolveWorkspace( // First try: look up workspace by name in wmill.yaml workspaces config const config = await readConfigFile({ warnIfMissing: false }); const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined; + // What wmill.yaml said to target, kept for the fallback below: a profile + // found by name can silently point somewhere else entirely. + let configuredTarget: { workspaceId: string; baseUrl: string } | undefined; if (wsEntry?.baseUrl) { const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry); let normalizedBaseUrl: string; @@ -231,6 +235,8 @@ async function tryResolveWorkspace( }; } + configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl }; + // Find matching profile by baseUrl + workspaceId const allProfs = await allWorkspaces(opts.configDir); const matching = allProfs.filter( @@ -283,6 +289,22 @@ async function tryResolveWorkspace( ), }; } + if ( + configuredTarget && + (e.workspaceId !== configuredTarget.workspaceId || + e.remote !== configuredTarget.baseUrl) + ) { + log.warnStderr( + colors.yellow( + `⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` + + ` Run: wmill workspace add ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}` + ) + ); + } + log.infoStderr( + `Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}` + ); (opts as any).__secret_workspace = e; return { isError: false, value: e }; } @@ -486,6 +508,8 @@ export async function resolveWorkspace( return process.exit(-1); } + let resolved: Workspace | undefined; + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { let existingWorkspace = await getWorkspaceByName( @@ -523,19 +547,45 @@ export async function resolveWorkspace( ); return process.exit(-1); } - return { + resolved = { ...existingWorkspace, token: opts.token, }; } } - return { + resolved ??= { remote: normalizedBaseUrl, workspaceId: opts.workspace, name: opts.workspace, token: opts.token, }; + + // --base-url pins the target, so wmill.yaml's `workspaces` block is never + // consulted and `--workspace` reaches the API as a workspace id. Name the + // id being sent, and the mapping being skipped, before the request 404s + // on an id the user never typed. + // Only an explicit `workspaceId:` is worth reporting: an entry without one + // maps the name to itself, leaving nothing to correct. + const yamlEntry = await peekWorkspaceEntry(opts.workspace); + const yamlWorkspaceId = yamlEntry?.workspaceId; + if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) { + log.warnStderr( + colors.yellow( + `⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` + + ` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.` + ) + ); + } + log.infoStderr( + `Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` + + (resolved.name !== resolved.workspaceId + ? `, profile '${resolved.name}')` + : ")") + ); + (opts as any).__secret_workspace = resolved; + return resolved; } else { log.infoStderr( colors.red( diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 5ac48753d6..d0301373da 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,13 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { + extractFolderPath, + isAppFolderMetadataFile, + isRawAppFolderMetadataFile, +} from "../utils/resource_folders.ts"; +import { deploysWithRawApp } from "../utils/app_files.ts"; +import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { userCache: Map; @@ -10,6 +17,35 @@ export interface PermissionedAsContext { userEmail: string; } +/** + * The whole-tree `sync push` and the single-item `push` commands must resolve + * ownership the same way, so both build the context here: a push that leaves it + * undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it. + * Undefined below syncBehavior v1, where that reassignment is the contract, and + * for a caller who is neither admin nor in `wm_deployers` the backend enforces + * it anyway — the flag on the context is what keeps the CLI from claiming + * otherwise. + */ +export async function buildPermissionedAsContext( + workspace: string, + syncBehavior: string | number | undefined +): Promise { + if (parseSyncBehavior(syncBehavior) < 1) { + return undefined; + } + const user = await wmill.whoami({ workspace }); + const userIsAdminOrDeployer = + user.is_admin || (user.groups ?? []).includes("wm_deployers"); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + ); + return { + userCache: new Map(), + userIsAdminOrDeployer, + userEmail: user.email, + }; +} + async function ensureUserCache( workspace: string, cache: Map @@ -59,6 +95,45 @@ function contentHasOnBehalfOf(content: string, typeStr: string): boolean { return false; } +type AppTypeStr = "app" | "raw_app"; + +function isAppTypeStr(typeStr: string): typeStr is AppTypeStr { + return typeStr === "app" || typeStr === "raw_app"; +} + +/** The app folder a file belongs to. `isAppFolderMetadataFile` and its raw twin + * match a literal `/`, unlike `extractFolderPath` — so normalize before either, + * or a Windows path takes a different branch from the same file on Linux. */ +function appFolderOf(path: string, typeStr: AppTypeStr): string { + return extractFolderPath(path, typeStr) ?? path; +} + +function toPosix(path: string): string { + return path.replaceAll("\\", "/"); +} + +/** App folders whose own metadata file is being added or deleted, which is how a + * whole app arrives or goes rather than being redeployed. Neither takes an owner + * over: a create has none yet, and a delete leaves none behind. */ +function appsArrivingOrLeaving(changes: Change[]): Set { + const folders = new Set(); + for (const change of changes) { + if (change.name === "edited") continue; + const path = toPosix(change.path); + if (!isAppFolderMetadataFile(path) && !isRawAppFolderMetadataFile(path)) { + continue; + } + let typeStr: string; + try { + typeStr = getTypeStrFromPath(path); + } catch { + continue; + } + if (isAppTypeStr(typeStr)) folders.add(appFolderOf(path, typeStr)); + } + return folders; +} + export async function preCheckPermissionedAs( changes: Change[], userEmail: string, @@ -71,6 +146,12 @@ export async function preCheckPermissionedAs( if (userIsAdminOrDeployer) return; const wouldChangeItems: { path: string; currentOwner: string }[] = []; + const addItem = (item: { path: string; currentOwner: string }) => { + if (!wouldChangeItems.some((i) => i.path === item.path)) { + wouldChangeItems.push(item); + } + }; + const arrivingOrLeaving = appsArrivingOrLeaving(changes); for (const change of changes) { let typeStr: string; @@ -80,6 +161,22 @@ export async function preCheckPermissionedAs( continue; } + // An app is redeployed whole by any change to any of the files it actually + // sends — added, edited or deleted alike — so its policy is rewritten + // regardless of what the file holds. Settled here, before the content the + // other kinds parse to find their owner, which an app has none of to parse. + if (isAppTypeStr(typeStr)) { + const path = toPosix(change.path); + const folder = appFolderOf(path, typeStr); + if ( + !arrivingOrLeaving.has(folder) && + (typeStr === "app" || deploysWithRawApp(path.slice(folder.length))) + ) { + addItem({ path: folder, currentOwner: "(app policy owner)" }); + } + continue; + } + if (change.name === "added") { const content = change.content; if (!content) continue; @@ -100,11 +197,6 @@ export async function preCheckPermissionedAs( const label = typeStr === "script" ? "(script owner)" : "(flow owner)"; wouldChangeItems.push({ path: change.path, currentOwner: label }); - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); } continue; } @@ -147,12 +239,6 @@ export async function preCheckPermissionedAs( } } continue; - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); - continue; } else if (typeStr === "schedule") { const match = beforeContent.match( /email:\s*["']?([^\s"']+)["']?/ diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 5271167a07..b8b71b1de9 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1037,12 +1037,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -1824,12 +1828,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -2705,12 +2713,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -4580,6 +4592,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# # Usage:: # # @task @@ -4587,10 +4615,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -4598,10 +4631,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -4609,7 +4644,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -4650,13 +4685,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # @@ -5535,7 +5574,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -6657,6 +6696,34 @@ A caught failure reads the same whether it came from a task or from a \`step()\` Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\` \`\`\`typescript +/** + * Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (\`fetch\`, \`fetch#2\`, \`fetch#3\`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and \`attempts\`. Retries with no + * \`delay\` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: \`2\` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a \`multiplier\` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; @@ -6665,6 +6732,7 @@ export interface TaskOptions { concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; } /** @@ -6682,9 +6750,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -6740,12 +6810,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -6828,6 +6902,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# # Usage:: # # @task @@ -6835,10 +6925,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -6846,10 +6941,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -6857,7 +6954,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -6898,13 +6995,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one \`\`wait_for_approval\`\` step. # @@ -7326,7 +7427,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory +Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file **Arguments:** \`[directory:string]\` @@ -8424,6 +8525,21 @@ properties: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + items: + type: string + description: 'Origins allowed to call this route cross-origin, matched against + the request''s Origin header (ignoring case) and echoed back on a match. When + set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin + the runnable returns via wm_headers. Use [''*''] to opt out of any restriction, + including the http_route_default_allowed_origins instance setting. An empty + list is not a configuration and resolves exactly as null does. When null, the + instance setting applies, or Access-Control-Allow-Origin: * if it is unset. + Ignored on a static website, which has no authentication of its own and so hands + out public files: restricting which browsers may read them protects nothing + while breaking cross-origin webfonts and fetches. A single-file static asset + is not exempt, since it can carry an authentication_method.' error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare path, diff --git a/cli/src/types.ts b/cli/src/types.ts index 9c517ff3c7..5713d06390 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -234,7 +234,7 @@ export async function pushObj( if (!rawAppName) { throw new Error(`Could not extract raw app name from path: ${p}`); } - await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs); + await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs, permissionedAsContext); } else if (typeEnding === "folder") { await pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { diff --git a/cli/src/utils/app_files.ts b/cli/src/utils/app_files.ts new file mode 100644 index 0000000000..ca1be1de0e --- /dev/null +++ b/cli/src/utils/app_files.ts @@ -0,0 +1,48 @@ +import { + APP_BACKEND_FOLDER, + RECORDINGS_FOLDER, +} from "../commands/app/app_metadata.ts"; + +/** Directories under a raw app that no push sends. */ +const NEVER_DEPLOYED_DIRS = new Set([ + "node_modules", + "dist", + ".claude", + "sql_to_apply", +]); + +/** Files under a raw app that no push sends. */ +const NEVER_DEPLOYED_FILES = new Set([ + "package-lock.json", + "DATATABLES.md", + "AGENTS.md", + "wmill.d.ts", +]); + +/** + * Whether an app-root-relative path (`/` separators, leading slash optional) + * reaches the server through any of a push's three channels: `raw_app.yaml` as + * metadata, the backend folder as runnables, the rest bundled by + * `collectAppFiles`. A path this rejects deploys nothing, so changing it is not + * a change to the app however much the sync diff lists it. `collectAppFiles` + * must not drift from this — it reads the same two sets. + */ +export function deploysWithRawApp(relativePath: string): boolean { + const segments = relativePath.split("/").filter(Boolean); + if (segments.length === 0) return false; + const name = segments[segments.length - 1]; + const dirs = segments.slice(0, -1); + // The sets below describe the bundle, which never walks into the backend + // folder — applying them there would strip a runnable whose file shares a + // name (`backend/wmill.d.ts` is the runnable `wmill.d`). Depth 1 because + // `loadRunnablesFromBackend` reads that folder's top level only. + if (dirs[0] === APP_BACKEND_FOLDER) return dirs.length === 1; + if (NEVER_DEPLOYED_FILES.has(name)) return false; + if (dirs.some((d) => NEVER_DEPLOYED_DIRS.has(d))) return false; + // Session recordings are written at the app root only, so an app with a + // `recordings/` component folder of its own still ships it. + if (dirs[0] === RECORDINGS_FOLDER) return false; + return true; +} + +export { NEVER_DEPLOYED_DIRS, NEVER_DEPLOYED_FILES }; diff --git a/cli/test/app_access_mode_unit.test.ts b/cli/test/app_access_mode_unit.test.ts new file mode 100644 index 0000000000..42a3fd082a --- /dev/null +++ b/cli/test/app_access_mode_unit.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { + executionModeForPush, + generatingPolicy, + markAccessFromPolicy, +} from "../src/commands/app/app.ts"; + +// The access mode is the one policy field a tracked app keeps; a pull then a push must +// deploy the mode that was pulled, guest included, not a default. +test("the access mode survives the app.yaml round trip", async () => { + const guest: any = { policy: { execution_mode: "guest" }, value: {} }; + markAccessFromPolicy(guest); + guest.policy = undefined; + expect(guest.guests).toBe(true); + expect(guest.public).toBeUndefined(); + expect(executionModeForPush(guest, undefined)).toBe("guest"); + await generatingPolicy( + guest, + "u/test/app", + executionModeForPush(guest, undefined), + undefined, + ); + expect(guest.policy.execution_mode).toBe("guest"); + + const anonymous: any = { policy: { execution_mode: "anonymous" }, value: {} }; + markAccessFromPolicy(anonymous); + anonymous.policy = undefined; + expect(anonymous.public).toBe(true); + expect(executionModeForPush(anonymous, undefined)).toBe("anonymous"); + + expect(executionModeForPush({ policy: { execution_mode: "publisher" } }, undefined)).toBe("publisher"); + expect(executionModeForPush({}, undefined)).toBe("publisher"); +}); + +// `viewer` is the narrowest mode — each runnable runs as the viewer, not as the +// app's identity — and the only one with no marker in the file, so both ways it +// can reach a push must survive rather than widen to `publisher`. +test("viewer is never widened to publisher by a push", () => { + // Carried over from the deployed app: a pull writes no marker for it. + expect(executionModeForPush({}, { execution_mode: "viewer" })).toBe("viewer"); + // Stated by the file, which is all a first push has to go on. + expect(executionModeForPush({ policy: { execution_mode: "viewer" } }, undefined)).toBe("viewer"); + // The open-access markers still win, in either direction. + expect(executionModeForPush({ public: true }, { execution_mode: "viewer" })).toBe("anonymous"); + expect(executionModeForPush({}, { execution_mode: "anonymous" })).toBe("publisher"); + // A stated mode is authoritative both ways: the carry-over is for a file that + // says nothing, so it must not pin a deployed app to `viewer` forever. + expect( + executionModeForPush({ policy: { execution_mode: "publisher" } }, { execution_mode: "viewer" }) + ).toBe("publisher"); +}); diff --git a/cli/test/base_url_workspace_resolution_unit.test.ts b/cli/test/base_url_workspace_resolution_unit.test.ts new file mode 100644 index 0000000000..66f618d113 --- /dev/null +++ b/cli/test/base_url_workspace_resolution_unit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { resolveWorkspace } from "../src/core/context.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; + +const BASE_URL = "http://localhost:9999/"; + +// --base-url pins the target: --workspace reaches the API as a workspace id and +// wmill.yaml is not consulted. The warning that says so may only peek at the +// file — readConfigFile() exits on an unsupported syncBehavior and throws on a +// malformed one, so resolving through it lets an unrelated config fail a +// command that never needed it. +async function withWmillYaml( + wmillYaml: string, + fn: (opts: GlobalOptions) => Promise +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_")); + const originalCwd = process.cwd(); + try { + await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml); + await writeFile(await getWorkspaceConfigFilePath(configDir), ""); + + process.chdir(repoDir); + await fn({ + configDir, + baseUrl: BASE_URL, + token: "sometoken", + workspace: "staging", + } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +describe("--base-url workspace resolution", () => { + const rejectedConfigs: [string, string][] = [ + ["an unsupported syncBehavior", "syncBehavior: v2\n"], + ["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'], + ]; + + for (const [label, wmillYaml] of rejectedConfigs) { + test(`resolves despite ${label}`, async () => { + await withWmillYaml(wmillYaml, async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + }); + }); + } + + test("a workspaces mapping never overrides the explicit workspace id", async () => { + await withWmillYaml( + "workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n", + async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + } + ); + }); +}); diff --git a/cli/test/deploy_on_behalf_of_unit.test.ts b/cli/test/deploy_on_behalf_of_unit.test.ts index 2214c0dc37..7e21b24832 100644 --- a/cli/test/deploy_on_behalf_of_unit.test.ts +++ b/cli/test/deploy_on_behalf_of_unit.test.ts @@ -1,11 +1,12 @@ import { expect, test } from "bun:test"; import { deployItem } from "../windmill-utils-internal/src/deploy.ts"; -// `deployItem` spreads the source item into the request body, and a script's/flow's -// on_behalf_of names a username that only exists in the source -// workspace. Sending it to the target pairs one workspace's principal with the other's -// email, which the backend rejects. Deleting the spread is an easy regression, so pin -// that the key never reaches the wire. +// `deployItem` spreads the source item into the request body, and the principal it carries +// (`on_behalf_of`, at the top level for a script or flow and inside the policy for an app) +// names a username that only exists in the source workspace. Sending it to the target pairs +// one workspace's principal with the other's email, which the backend rejects. Deleting the +// spread is an easy regression, so pin that the principal never reaches the wire while the +// caller's chosen address does. function recordingProvider(captured: [string, any][], flowExists: boolean) { const source = { on_behalf_of_email: "alice@corp", @@ -32,6 +33,19 @@ function recordingProvider(captured: [string, any][], flowExists: boolean) { }), createScript: async (p: any) => void captured.push(["createScript", p.requestBody]), + existsApp: async () => false, + getAppByPath: async () => ({ + path: "f/x/a", + summary: "", + value: {}, + raw_app: false, + policy: { + execution_mode: "publisher", + on_behalf_of: "u/alice", + on_behalf_of_email: "alice@corp", + }, + }), + createApp: async (p: any) => void captured.push(["createApp", p.requestBody]), } as any; } @@ -65,19 +79,29 @@ test("deployItem: never sends the source workspace's on_behalf_of", async () => "dst", "alice@corp", ); + await deployItem( + recordingProvider(captured, false), + "app" as any, + "f/x/a", + "src", + "dst", + "alice@corp", + ); expect(captured.map(([fn]) => fn)).toEqual([ "createFlow", "updateFlow", "createScript", + "createApp", ]); - for (const [, body] of captured) { - // The email is still overridden with the caller's choice... - expect(body.on_behalf_of_email).toBe("alice@corp"); + for (const [name, body] of captured) { expect(body.preserve_on_behalf_of).toBe(true); + // Both surfaces spell it `on_behalf_of`; only its nesting differs — an app carries the + // identity inside its policy, the others at the top level. + const identity = name === "createApp" ? body.policy : body; + // The email is still overridden with the caller's choice... + expect(identity.on_behalf_of_email).toBe("alice@corp"); // ...while the principal is dropped, so the backend derives the target's own. - expect( - "on_behalf_of" in JSON.parse(JSON.stringify(body)), - ).toBe(false); + expect("on_behalf_of" in JSON.parse(JSON.stringify(identity))).toBe(false); } }); diff --git a/cli/test/lint_orphan_metadata_unit.test.ts b/cli/test/lint_orphan_metadata_unit.test.ts new file mode 100644 index 0000000000..7066d10dca --- /dev/null +++ b/cli/test/lint_orphan_metadata_unit.test.ts @@ -0,0 +1,166 @@ +import { expect, test, describe } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import * as path from "node:path"; +import { runLint } from "../src/commands/lint/lint.ts"; + +const WMILL_YAML = "defaultTs: bun\nincludes:\n - f/**\nexcludes: []\n"; +const METADATA = "summary: test\nlock: ''\nschema:\n properties: {}\n"; + +async function write(dir: string, rel: string, content: string) { + const full = path.join(dir, rel); + await mkdir(path.dirname(full), { recursive: true }); + await writeFile(full, content, "utf-8"); +} + +/** + * Runs `fn` with a sync root at `/`, from which lint resolves + * every walked path. The name is a parameter because it is load-bearing: the + * folder suffixes lint classifies by (`.app`, `__mod`, …) are matched anywhere + * in a path, so a root carrying one must not change what lint reports. + */ +async function withSyncRoot( + rootName: string, + fn: (syncRoot: string) => Promise, + opts: { runFromParent?: boolean } = {}, +): Promise { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "wmill_lint_orphan_")); + const syncRoot = path.join(tempDir, rootName); + const originalCwd = process.cwd(); + try { + await write(syncRoot, "wmill.yaml", WMILL_YAML); + process.chdir(opts.runFromParent ? tempDir : syncRoot); + await fn(syncRoot); + } finally { + process.chdir(originalCwd); + await rm(tempDir, { recursive: true }); + } +} + +describe("orphan script metadata", () => { + test("reports metadata with no content file, with locks not required", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/paired.py", "def main():\n pass\n"); + await write(syncRoot, "f/paired.script.yaml", METADATA); + await write(syncRoot, "f/orphan.script.yaml", METADATA); + await write(syncRoot, "f/orphan_json.script.json", "{}\n"); + await write(syncRoot, "f/orphan_lock.script.lock", "some-dep==1.0.0\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.exitCode).toBe(1); + expect(report.issues.map((i) => i.path).sort()).toEqual([ + "f/orphan.script.yaml", + "f/orphan_json.script.json", + "f/orphan_lock.script.lock", + ]); + expect(report.issues[0].target).toBe("script"); + expect(report.issues[0].errors[0]).toContain("No script file found next to"); + }); + }); + + test("reports a module folder's own metadata with no content file", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/orphan__mod/script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual([ + "f/orphan__mod/script.yaml", + ]); + + // Linting the module folder itself: the walked paths no longer carry the + // `__mod/` boundary that says this is a module's metadata. + const inFolder = await runLint( + {} as any, + path.join(syncRoot, "f/orphan__mod"), + ); + + expect(inFolder.issues.map((i) => i.path)).toEqual(["script.yaml"]); + }); + }); + + test("reports orphans under a sync root named like a resource folder", async () => { + await withSyncRoot("acme.app", async (syncRoot) => { + await write(syncRoot, "f/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual(["f/orphan.script.yaml"]); + }); + }); + + test("does not report a paired module under a sync root named like a module folder", async () => { + // Run from OUTSIDE the checkout, the one invocation whose paths carry the + // root's own name: nothing above the sync root may be classified. + await withSyncRoot( + "repo__mod", + async (syncRoot) => { + await write(syncRoot, "f/example__mod/script.yaml", METADATA); + await write( + syncRoot, + "f/example__mod/script.ts", + "export function main() {}\n", + ); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("does not report a non-dotted folder resource's child", async () => { + // The dotted/non-dotted setting is read from the invocation directory, so + // an explicit target configured the other way must still be recognized. + await withSyncRoot( + "repo", + async (syncRoot) => { + await write(syncRoot, "f/a__raw_app/raw_app.yaml", "value: {}\n"); + await write(syncRoot, "f/a__raw_app/backend/config.script.lock", "x\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("keeps the reported path whole when the lint target is a path segment", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/conf/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, path.join(syncRoot, "f")); + + expect(report.issues.map((i) => i.path)).toEqual([ + "conf/orphan.script.yaml", + ]); + expect(report.issues[0].errors[0]).toContain("conf/orphan.script.yaml"); + }); + }); + + test("does not report a fileset child spelled like script metadata", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/data.resource.yaml", "value: {}\n"); + await write(syncRoot, "f/data.fileset/config.script.yaml", "a: 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); + + test("does not report a dbt project whose optional descriptor is absent", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/proj.script.yaml", METADATA); + await write(syncRoot, "f/proj__dbt/dbt_project.yml", "name: proj\n"); + await write(syncRoot, "f/proj__dbt/models/a.sql", "select 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); +}); diff --git a/cli/test/precheck_permissioned_as_apps_unit.test.ts b/cli/test/precheck_permissioned_as_apps_unit.test.ts new file mode 100644 index 0000000000..4a224a43c0 --- /dev/null +++ b/cli/test/precheck_permissioned_as_apps_unit.test.ts @@ -0,0 +1,147 @@ +/** + * The pre-check is what stops a push from silently reassigning an item's run-as + * user. Raw apps were missing from it, so the one kind whose file records no + * policy at all was also the one that changed owner without a word. + */ + +import { expect, test } from "bun:test"; +import { preCheckPermissionedAs } from "../src/core/permissioned_as.ts"; + +/** Non-interactive and without the override flag, the pre-check exits rather + * than reassigning silently — so a thrown exit is the signal it fired. */ +type Shape = "edited" | "added" | "deleted"; + +function change(path: string, name: Shape = "edited") { + return { name, path, before: "summary: x\n", content: "summary: x\n" }; +} + +async function precheck( + changes: ReturnType[], +): Promise { + const exit = process.exit; + let code: number | undefined; + (process as any).exit = (c?: number) => { + code = c; + throw new Error(`exit:${c}`); + }; + const logged: string[] = []; + const err = console.error; + console.error = (...a: unknown[]) => void logged.push(a.join(" ")); + try { + await preCheckPermissionedAs(changes, "pusher@corp", false, false, false); + } catch (e) { + if (!String(e).startsWith("Error: exit:")) throw e; + } finally { + (process as any).exit = exit; + console.error = err; + } + return code === undefined ? undefined : logged.join("\n"); +} + +test("a raw-app push warns the non-deployer it will take over the run-as user", async () => { + const message = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + + expect(message).toBeDefined(); + expect(message).toContain("f/test/myapp.raw_app"); + expect(message).toContain("pusher@corp"); +}); + +// Deleting one file re-pushes the whole app rather than deleting it, so the +// takeover happens there too. +test("deleting one of an app's files warns like editing one", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/gone.tsx", "deleted"), + ]); + + expect(message).toContain("f/test/myapp.raw_app"); +}); + +// The metadata file going with it means the app itself is created or removed — +// neither takes an owner over. +test("an app arriving or leaving whole is not a takeover", async () => { + const created = await precheck([ + change("f/test/new.raw_app/raw_app.yaml", "added"), + change("f/test/new.raw_app/index.tsx", "added"), + ]); + const removed = await precheck([ + change("f/test/old.raw_app/raw_app.yaml", "deleted"), + change("f/test/old.raw_app/index.tsx", "deleted"), + ]); + + expect(created).toBeUndefined(); + expect(removed).toBeUndefined(); +}); + +// An app carries no owner in its files, so nothing about it depends on their +// content — an empty one redeploys it exactly like any other. +test("an empty file still counts as a change to the app", async () => { + const added = await precheck([ + { name: "added", path: "f/test/myapp.raw_app/blank.ts", content: "" }, + ]); + const edited = await precheck([ + { name: "edited", path: "f/test/myapp.raw_app/blank.ts", before: "" }, + ]); + + expect(added).toContain("f/test/myapp.raw_app"); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `extractFolderPath` normalizes separators but the metadata predicates match a +// literal `/`, so a Windows path must not take a different branch. +test("a Windows path classifies the same as its posix twin", async () => { + const created = await precheck([ + change("f\\test\\new.raw_app\\raw_app.yaml", "added"), + change("f\\test\\new.raw_app\\index.tsx", "added"), + ]); + const edited = await precheck([ + change("f\\test\\myapp.raw_app\\index.tsx"), + ]); + + expect(created).toBeUndefined(); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `collectAppFiles` never sends these, and the sync diff never stops listing +// them (nothing uploads them, so they stay "added" forever) — so warning on one +// would gate every push of a scaffolded app on the override flag. +test("a file the push never sends is not a change to the app", async () => { + const artifacts = await precheck([ + change("f/test/myapp.raw_app/AGENTS.md", "added"), + change("f/test/myapp.raw_app/sql_to_apply/a.sql", "added"), + change("f/test/myapp.raw_app/node_modules/dep/index.js", "added"), + change("f/test/myapp.raw_app/recordings/r.json", "added"), + change("f/test/myapp.raw_app/package-lock.json"), + change("f/test/myapp.raw_app/wmill.d.ts"), + // Only the backend folder's *top level* is a runnable; nothing reads deeper, + // so the depth limit is what keeps a `backend/node_modules/` from becoming + // the perpetual diff this predicate exists to remove. + change("f/test/myapp.raw_app/backend/node_modules/dep/index.js", "added"), + ]); + // The three channels a push does send through: bundled file, metadata, runnable. + const sent = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + const meta = await precheck([change("f/test/myapp.raw_app/raw_app.yaml")]); + const runnable = await precheck([change("f/test/myapp.raw_app/backend/a.ts")]); + // The runnable channel is not the bundle: the bundle's name exclusions don't + // reach into it, so a runnable file sharing one of those names still deploys. + const namesake = await precheck([ + change("f/test/myapp.raw_app/backend/wmill.d.ts"), + ]); + + expect(artifacts).toBeUndefined(); + expect(sent).toContain("f/test/myapp.raw_app"); + expect(meta).toContain("f/test/myapp.raw_app"); + expect(runnable).toContain("f/test/myapp.raw_app"); + expect(namesake).toContain("f/test/myapp.raw_app"); +}); + +test("an app is listed once however many of its files changed", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/index.tsx"), + change("f/test/myapp.raw_app/raw_app.yaml"), + change("f/test/myapp.raw_app/backend/a.ts"), + change("f/test/low.app/app.yaml"), + change("f/test/low.app/inline.ts"), + ]); + + expect(message).toContain("2 item(s)"); +}); diff --git a/cli/test/raw_app_push_policy_unit.test.ts b/cli/test/raw_app_push_policy_unit.test.ts new file mode 100644 index 0000000000..fbbc7b55c0 --- /dev/null +++ b/cli/test/raw_app_push_policy_unit.test.ts @@ -0,0 +1,142 @@ +/** + * `raw_app.yaml` records none of the policy but the access-mode markers, so a + * push that regenerated the whole policy reset the deploy drawer's settings — + * run-as identity, sandbox isolation — to the pushing user's. Pin that the + * deployed policy is carried over, that a first push still starts from what the + * file states, and that the markers still close a deployed open app back down. + */ + +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { mkdtemp, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let calls: any[] = []; +let deployedPolicy: any; +/** No app deployed at the path: `getAppByPath` 404s and the push creates one. */ +let deployed = true; + +// Stub only what no other in-process suite imports, and treat a stub as +// permanent for the run (see "Module mocks" in cli/TESTING.md). These three API +// functions qualify — nothing else in `test/` imports them. `bundle.ts` did not: +// stubbing it left `raw_app_svelte_plugin_unit.test.ts` asserting against an +// empty bundle, which an `afterAll` hand-back did not prevent. So the real +// bundler runs instead, on the app each push writes below. +const realServices = await import("../gen/services.gen.ts"); + +mock.module("../gen/services.gen.ts", () => ({ + ...realServices, + getAppByPath: async () => { + if (!deployed) throw new Error("not found"); + return { + path: "f/test/raw", + summary: "raw", + value: { files: {}, runnables: {} }, + policy: deployedPolicy, + }; + }, + updateAppRaw: async (a: unknown) => { + calls.push(a); + }, + createAppRaw: async (a: unknown) => { + calls.push(a); + }, +})); + +// Belt and braces: nothing else in-process calls these, and a hand-back is not +// what makes that safe. +afterAll(() => { + mock.module("../gen/services.gen.ts", () => realServices); +}); + +const { pushRawApp } = await import("../src/commands/app/raw_apps.ts"); + +const ADMIN = { + userCache: new Map(), + userIsAdminOrDeployer: true, + userEmail: "deployer@windmill.dev", +}; + +async function push(yamlTail: string, admin = true): Promise { + calls = []; + const dir = await mkdtemp(join(tmpdir(), "windmill_raw_push_")); + await writeFile( + join(dir, "raw_app.yaml"), + `summary: raw\nrunnables: {}\n${yamlTail}`, + "utf-8", + ); + // Any file the remote doesn't have, so the push isn't short-circuited as + // up to date. It is also the bundler's entry point, so it has to compile. + await writeFile(join(dir, "index.tsx"), "export default 1\n", "utf-8"); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ name: "app", private: true }), + "utf-8", + ); + // `ensureNodeModules` only checks the directory is there; borrowing the CLI's + // own skips an npm install per push. + await symlink(join(process.cwd(), "node_modules"), join(dir, "node_modules")); + await pushRawApp("w", "f/test/raw", dir, undefined, "bun", admin ? ADMIN : undefined); + expect(calls).toHaveLength(1); + return calls[0].formData.app; +} + +beforeEach(() => { + deployed = true; + deployedPolicy = { + on_behalf_of: "u/svc", + on_behalf_of_email: "svc@corp", + sandbox: true, + frontend_sdk_scopes: ["jobs:run"], + execution_mode: "anonymous", + // Legacy v1 grants: the backend folds them into v2 at run time, so keeping + // them would keep granting runnables a push has removed. + triggerables: { "script/f/test/gone": {} }, + triggerables_v2: { "a:script/f/test/gone": {} }, + }; +}); + +test("a raw-app push keeps the deployed run-as and sandbox settings", async () => { + const body = await push("public: true\n"); + + expect(body.policy.on_behalf_of).toBe("u/svc"); + expect(body.policy.on_behalf_of_email).toBe("svc@corp"); + expect(body.preserve_on_behalf_of).toBe(true); + expect(body.policy.sandbox).toBe(true); + expect(body.policy.frontend_sdk_scopes).toEqual(["jobs:run"]); + expect(body.policy.execution_mode).toBe("anonymous"); + expect(body.policy.triggerables).toBeUndefined(); + expect(body.policy.triggerables_v2).toEqual({}); +}); + +test("a raw-app push without the marker closes an anonymous app back down", async () => { + const body = await push(""); + + expect(body.policy.execution_mode).toBe("publisher"); +}); + +test("a push that may not claim the deployed identity doesn't send it", async () => { + const body = await push("", false); + + expect(body.preserve_on_behalf_of).toBeUndefined(); + // Not just the flag: the identity itself stays off the wire, so no server can + // deploy this push under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + // Everything the pusher is entitled to carry over still comes along. + expect(body.policy.sandbox).toBe(true); +}); + +test("a first raw-app push deploys the policy its file states", async () => { + deployed = false; + const body = await push( + "policy:\n sandbox: true\n on_behalf_of: u/impostor\n on_behalf_of_email: impostor@corp\n", + ); + + expect(body.policy.sandbox).toBe(true); + // A repo doesn't get to pick who an app runs as: the identity never reaches + // the wire, so no server can be talked into deploying under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + expect(body.preserve_on_behalf_of).toBeUndefined(); +}); diff --git a/cli/test/schedule_push_permissioned_as_unit.test.ts b/cli/test/schedule_push_permissioned_as_unit.test.ts new file mode 100644 index 0000000000..17a1731494 --- /dev/null +++ b/cli/test/schedule_push_permissioned_as_unit.test.ts @@ -0,0 +1,143 @@ +/** + * Regression guard: the standalone `wmill schedule push` must resolve ownership + * the same way `wmill sync push` does. It only preserves the remote's + * `permissioned_as` when the command hands `pushSchedule` a context, so a push + * that builds none silently reassigns the schedule to whoever ran it. + */ + +import { expect, test, describe, afterAll, beforeEach, mock } from "bun:test"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let updateScheduleCalls: any[] = []; +let remotePermissionedAs: string | undefined = "u/svc"; + +const REMOTE_SCHEDULE = () => ({ + path: "u/admin/sched", + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: "u/admin/script", + is_flow: false, + args: {}, + enabled: false, + summary: "before", + permissioned_as: remotePermissionedAs, +}); + +// A module mock is process-global and outlives the file that installs it, and +// `mock.restore()` does not undo one: every mocked module has to be handed back +// its real exports here, or whichever file `bun test` happens to run next gets +// this file's stubs. +const realModules: [string, Record][] = []; +async function mockModule( + specifier: string, + factory: (real: Record) => Record +): Promise { + const real = { ...((await import(specifier)) as Record) }; + realModules.push([specifier, real]); + mock.module(specifier, () => factory(real)); +} + +afterAll(() => { + for (const [specifier, real] of realModules) { + mock.module(specifier, () => real); + } +}); + +await mockModule("../gen/services.gen.ts", () => ({ + getSchedule: async () => REMOTE_SCHEDULE(), + updateSchedule: async (a: unknown) => { + updateScheduleCalls.push(a); + }, + whoami: async () => ({ + email: "deployer@windmill.dev", + username: "deployer", + is_admin: true, + groups: [], + }), +})); + +await mockModule("../src/core/context.ts", (real) => ({ + ...real, + resolveWorkspace: async () => ({ + workspaceId: "w", + name: "w", + remote: "http://localhost/", + token: "t", + }), +})); + +await mockModule("../src/core/auth.ts", (real) => ({ + ...real, + requireLogin: async () => ({}), +})); + +const scheduleCommand = (await import("../src/commands/schedule/schedule.ts")) + .default; + +async function pushIn(wmillYamlTail: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_")); + await writeFile( + join(dir, "wmill.yaml"), + `defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`, + "utf-8" + ); + await writeFile( + join(dir, "sched.schedule.yaml"), + `schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`, + "utf-8" + ); + + const cwd = process.cwd(); + process.chdir(dir); + try { + await scheduleCommand.parse([ + "push", + "sched.schedule.yaml", + "u/admin/sched", + ]); + } finally { + process.chdir(cwd); + } +} + +describe("wmill schedule push ownership", () => { + beforeEach(() => { + updateScheduleCalls = []; + remotePermissionedAs = "u/svc"; + }); + + test("keeps the remote's permissioned_as under syncBehavior v1", async () => { + await pushIn("syncBehavior: v1\n"); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.summary).toBe("after"); + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + // The entry to read is the one matching the workspace being pushed to, not + // the top level: a repo that varies settings per workspace puts syncBehavior + // under `overrides` and nowhere else. + test("reads syncBehavior from the target workspace's overrides", async () => { + await pushIn( + `workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n` + ); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + test("leaves ownership to the backend below syncBehavior v1", async () => { + await pushIn(""); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBeUndefined(); + expect(body.preserve_permissioned_as).toBeUndefined(); + }); +}); diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts index e9ea167ffc..4854198d64 100644 --- a/cli/test/sync_push_auto_metadata_repro.test.ts +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -168,7 +168,7 @@ test( // Customer scenario: a barrel file (f/lib/errors/index.ts) re-exports from // siblings (./types.ts, ./WorkflowError.ts, ...). An importer in a different // folder imports from the barrel. On a fresh DB, the dep job for the importer -// fetches index.ts via raw_unpinned + temp_script_hash, but bun's resolver +// fetches index.ts via raw + temp_script_hash, but bun's resolver // then has to resolve the barrel's *sibling* imports — and those need to be // in TEMP_SCRIPT_REFS too. test( diff --git a/cli/test/variable_resource_push.test.ts b/cli/test/variable_resource_push.test.ts index e6bc84c326..b77ed48b50 100644 --- a/cli/test/variable_resource_push.test.ts +++ b/cli/test/variable_resource_push.test.ts @@ -361,6 +361,126 @@ describe("variable", () => { expect(content).toContain("is_secret: false"); }); }); + + test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/perms_var_${uniqueId}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: varPath, + value: "perms_test_value", + is_secret: false, + description: "Variable for extra_perms test", + }), + } + ); + expect(createResp.status).toBeLessThan(300); + await createResp.text(); + + const aclResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/acls/add/variable/${varPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ owner: "g/all", write: true }), + } + ); + expect(aclResp.status).toBeLessThan(300); + await aclResp.text(); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`, + "utf-8" + ); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir + ); + expect(pullResult.code).toEqual(0); + + const localPath = join(tempDir, `${varPath}.variable.yaml`); + const pulled = await readFile(localPath, "utf-8"); + expect(pulled).toContain("extra_perms:"); + expect(pulled).toContain("g/all: true"); + + const beforeResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const before = await beforeResp.json(); + + // Perm-only edit: downgrade the grant to read. + await writeFile( + localPath, + pulled.replace("g/all: true", "g/all: false"), + "utf-8" + ); + + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + const afterResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const after = await afterResp.json(); + expect(after.extra_perms).toEqual({ "g/all": false }); + // Routed through /acls/* rather than update_variable, so the row itself + // is untouched. + expect(after.edited_at).toEqual(before.edited_at); + expect(after.value).toEqual("perms_test_value"); + + // A yaml with no extra_perms field at all is "no opinion": a checkout + // that predates ACL sync must never revoke UI-managed grants. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`, + "utf-8" + ); + const noOpinionResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(noOpinionResult.code).toEqual(0); + + const noOpinionApiResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + expect((await noOpinionApiResp.json()).extra_perms).toEqual({ + "g/all": false, + }); + + // An owner present remotely but absent from a *present* map is revoked — + // the one direction that can destroy a grant. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`, + "utf-8" + ); + const revokeResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(revokeResult.code).toEqual(0); + + const finalResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const final = await finalResp.json(); + expect(final.extra_perms).toEqual({}); + }); + }); }); // ============================================================================= diff --git a/cli/test/workspace_key_filename_integration.test.ts b/cli/test/workspace_key_filename_integration.test.ts index ae22f191bf..a2e731eae5 100644 --- a/cli/test/workspace_key_filename_integration.test.ts +++ b/cli/test/workspace_key_filename_integration.test.ts @@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml"; import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts"; import { findResourceFile } from "../src/commands/script/script.ts"; -import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts"; -import type { SyncOptions } from "../src/core/conf.ts"; +import { + resolveWsNameForConfigFromFlags, + type SyncOptions, +} from "../src/core/conf.ts"; // Integration tests covering the bug where workspace-specific filenames used // the raw git branch name instead of the wmill.yaml workspace config key. diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index 047d50ee78..ac0b775e86 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -506,10 +506,23 @@ export async function deployItem( }, }); } else if (kind === "app" || kind === "raw_app") { - const app = await provider.getAppByPath({ + const rawApp = await provider.getAppByPath({ workspace: workspaceFrom, path, }); + // See the flow branch: a source-workspace principal is never valid here, and the + // policy carries the app's in `on_behalf_of`. Clearing it lets the backend derive + // the target's own from the address. A group travels as its synthetic + // `group-*@windmill.dev` address, which an admin-created account holding it would + // win: known and accepted, see `users::permissioned_as_from_email` in the backend. + const app = { + ...rawApp, + policy: { + ...rawApp.policy, + on_behalf_of: undefined, + on_behalf_of_email: onBehalfOf, + }, + }; if (alreadyExists) { if (app.raw_app) { const secret = await provider.getPublicSecretOfLatestVersionOfApp({ diff --git a/docs/app-policy-email-removal.md b/docs/app-policy-email-removal.md new file mode 100644 index 0000000000..fb4cf100fd --- /dev/null +++ b/docs/app-policy-email-removal.md @@ -0,0 +1,47 @@ +# Removing `policy.on_behalf_of_email` + +An app's identity is `policy.on_behalf_of`; the address beside it is a function of that +principal. `on_behalf_of_email` is no longer required — a policy carrying only a principal +executes, deriving the address — but it is still written on every save, and that is the only +thing holding it in place. + +## The gate + +`get_on_behalf_of` gained its derive-when-absent fallback in **1.810**. Every replica before that +*requires* the key and errors outright without it, so it would 400 every anonymous, publisher and +guest app saved by a newer one. A rolling deploy runs both versions at once, which is why the +write stays until no replica older than 1.810 can be live — in practice, once +`MIN_KEEP_ALIVE_VERSION` (`windmill-common/src/min_version.rs`) has passed it. + +There is no `MIN_VERSION_*` constant for this and it does not need one: those exist to gate +behavior at runtime or to trip the build when a constraint expires, and nothing here does either. +The key is written unconditionally, so no replica ever meets its absence until someone follows +the steps below. + +## Step 1 — stop writing the key + +- `stored_on_behalf_of_email` in `windmill-api/src/apps.rs`, and the `create_app` / + `update_app_internal` call sites that store what it returns. +- The CLI and frontend workspace-deploy paths (`cli/windmill-utils-internal/src/deploy.ts`, + `frontend/src/lib/utils_workspace_deploy.ts`). These send the address *instead of* a principal + for a cross-workspace deploy, which is the one case where it is the only identity available — + so this is "stop sending it once the target resolves a principal itself", not a deletion. + +Policies written before this keep their key and keep being read from it; they agree with their +principal, so nothing has to strip them. + +## Step 2 — drop the field + +Remove `on_behalf_of_email` from `Policy` and the fallback in `get_on_behalf_of`, which then +always derives. Optionally strip the key from stored policies. + +This can ship with step 1. It is written separately because step 1 alone is revertible without +touching stored data or the response schema, and because the gate above is what makes either +step safe — nothing about step 2 needs its own waiting period. + +## Why the address is not derived on read + +Read paths return the stored address verbatim rather than recomputing it. Deriving on read means +resolving a principal that, for a draft, is caller-controlled — which turns the read into an +oracle for addresses the caller cannot otherwise see, and leaves a principal that resolves to +nobody with no address at all. Both were live defects while the read paths did derive. diff --git a/docs/clone_repo_and_upload_to_instance_storage.bun.ts b/docs/clone_repo_and_upload_to_instance_storage.bun.ts deleted file mode 100644 index 485ac0c766..0000000000 --- a/docs/clone_repo_and_upload_to_instance_storage.bun.ts +++ /dev/null @@ -1,390 +0,0 @@ -import * as wmillclient from "windmill-client"; -import { basename, join } from "node:path"; -import { existsSync, rmSync } from "fs"; -import process from "process"; -import { spawn } from 'child_process'; -import * as fs_async from 'fs/promises'; -import * as fs from 'node:fs'; - -const UPLOAD_CONCURRENCY = 16; -const CLONE_MARKER_FILE = ".windmill_clone_complete"; - -type GitRepository = { - url: string; - branch: string; - folder: string; - gpg_key: any; - is_github_app: boolean; -}; - -export async function main( - resource_path: string, - workspace: string, - git_ssh_identity?: string[], - commit?: string -) { - let clonedRepoPath: string | undefined; - - try { - console.log("Starting git clone and Blob storage upload process"); - - // Get the git repository resource - const repo_resource: GitRepository = await wmillclient.getResource(resource_path); - - const cwd = process.cwd(); - - if (git_ssh_identity) { - process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity) - } - - // Handle GitHub App authentication if needed - if (repo_resource.is_github_app) { - const token = await get_gh_app_token(); - repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token); - } - - process.env["HOME"] = "."; - process.env.GIT_TERMINAL_PROMPT = "0"; - - // Clone the repository - const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit); - clonedRepoPath = join(cwd, repo_name); - - // Remove .git directory to avoid uploading git history - const gitDir = join(clonedRepoPath, ".git"); - if (existsSync(gitDir)) { - rmSync(gitDir, { recursive: true, force: true }); - console.log("Removed .git directory"); - } - - // Upload to S3 - const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`; - const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace); - - return { - success: true, - message: "Repository cloned and uploaded to S3 successfully", - s3_path: s3Path, - commit_hash: commitHash, - file_count: fileCount, - }; - - } catch (error) { - console.error("Error in git clone and upload:", error); - throw error; - } finally { - // Clean up cloned repository - if (clonedRepoPath && existsSync(clonedRepoPath)) { - rmSync(clonedRepoPath, { recursive: true, force: true }); - console.log("Cleaned up cloned repository"); - } - } -} - -async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise { - const sshIdFiles = await Promise.all( - git_ssh_identity.map(async (varPath, i) => { - const filePath = join(cwd, `./ssh_id_priv_${i}`); - - try { - // Get variable value using windmill - let content = await wmillclient.getVariable(varPath); - content += '\n'; - - // Write file with content - await fs_async.writeFile(filePath, content, { encoding: 'utf8' }); - - // Set file permissions to 0o600 (read/write for owner only) - await fs_async.chmod(filePath, 0o600); - - // Escape single quotes for shell command - const escapedPath = filePath.replace(/'/g, "'\\''"); - return ` -i '${escapedPath}'`; - } catch (error) { - console.error( - `Variable ${varPath} not found for git ssh identity: ${error}` - ); - return ''; - } - }) - ); - - const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`; - return gitSshCmd; -} - -async function git_clone( - cwd: string, - repo_resource: GitRepository, - commit?: string, -): Promise<{ repo_name: string; commitHash: string }> { - if (commit) { - return git_clone_at_commit(cwd, repo_resource, commit); - } else { - return git_clone_at_latest(cwd, repo_resource); - } -} - -async function git_clone_at_commit( - cwd: string, - repo_resource: GitRepository, - commit: string, -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const repoPath = join(cwd, repo_name); - await fs_async.mkdir(repoPath, { recursive: true }); - - process.chdir(repoPath); - - let args = ['init', '--quiet'] - if (branch) { - args.push(`--initial-branch=${branch}`) - } - await runCommand(undefined, 'git', ...args); - - await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url); - - await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit); - - await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD'); - - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function git_clone_at_latest( - cwd: string, - repo_resource: GitRepository -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - // Handle Azure DevOps token if needed - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const args = ["clone", "--quiet", "--depth", "1"]; - if (subfolder !== "") args.push("--sparse"); - if (branch !== "") args.push("--branch", branch); - args.push(repo_url, repo_name); - - await runCommand(-1, "git", ...args); - - const fullPath = join(cwd, repo_name); - process.chdir(fullPath); - - if (subfolder !== "") { - await runCommand(undefined, "git", "sparse-checkout", "add", subfolder); - const subfolderPath = join(fullPath, subfolder); - - if (!existsSync(subfolderPath)) { - throw new Error(`Subfolder ${subfolder} does not exist.`); - } - - process.chdir(subfolderPath); - } - - // Get the commit hash - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function uploadDirectoryToS3( - directoryPath: string, - s3BasePath: string, - workspace: string, -): Promise { - console.log(`Uploading ${directoryPath} -> ${s3BasePath}`); - - // Walk once into a flat task list so we can drive a bounded-concurrency pool. - const tasks: { localPath: string; s3Key: string }[] = []; - function walk(dir: string, s3Path: string) { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name); - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - walk(fullPath, s3Key); - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }); - } - } - } - walk(directoryPath, s3BasePath); - console.log(`Discovered ${tasks.length} files to upload`); - - let nextIndex = 0; - let uploaded = 0; - let lastReport = 0; - async function worker() { - while (true) { - const idx = nextIndex++; - if (idx >= tasks.length) return; - const { localPath, s3Key } = tasks[idx]; - const fileContent = fs.readFileSync(localPath); - const blob = new Blob([fileContent], { type: 'application/octet-stream' }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: s3Key, - requestBody: blob, - }); - uploaded++; - if (uploaded - lastReport >= 25 || uploaded === tasks.length) { - lastReport = uploaded; - console.log(`Uploaded ${uploaded} / ${tasks.length} files`); - } - } - } - await Promise.all( - Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker()) - ); - - // Marker is the LAST write — its presence is what the viewer checks for. - const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`; - const markerBody = JSON.stringify({ - completed_at: new Date().toISOString(), - file_count: tasks.length, - }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: markerKey, - requestBody: new Blob([markerBody], { type: 'application/json' }), - }); - console.log(`Wrote completion marker: ${markerKey}`); - - return tasks.length; -} - -function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise { - const nargs = secret_position != undefined ? args.slice() : args; - if (secret_position && secret_position < 0) - secret_position = nargs.length - 1 + secret_position; - - let secret: string | undefined = undefined; - if (secret_position != undefined) { - nargs[secret_position] = "***"; - secret = args[secret_position]; - } - console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`); - - return new Promise((resolve, reject) => { - const process = spawn(cmd, args); - - let stdout = ''; - let stderr = ''; - - process.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - process.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - process.on('error', (error) => { - let errorString = error.toString(); - if (secret) errorString = errorString.replace(secret, "***"); - console.log(`Shell command FAILED: ${cmd}`, errorString); - const e = new Error( - `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}` - ); - reject(e); - }); - - process.on('close', (code) => { - if (stdout.length > 0) { - console.log("Shell stdout:", stdout); - } - if (stderr.length > 0) { - console.log("Shell stderr:", stderr); - } - if (code === 0) { - console.log(`Shell command completed successfully: ${cmd}`); - resolve(stdout); - } else { - reject(new Error(`Command failed with code ${code}: ${stderr}`)); - } - }); - }); -} - -async function get_gh_app_token() { - const workspace = process.env["WM_WORKSPACE"]; - const jobToken = process.env["WM_TOKEN"]; - const baseUrl = - process.env["BASE_INTERNAL_URL"] ?? - process.env["BASE_URL"] ?? - "http://localhost:8000"; - const url = `${baseUrl}/api/w/${workspace}/github_app/token`; - - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${jobToken}`, - }, - body: JSON.stringify({ job_token: jobToken }), - }); - - if (!response.ok) { - const errorBody = await response.text().catch(() => ""); - throw new Error(`GitHub App token error (${response.status}): ${errorBody || response.statusText}`); - } - const data = await response.json(); - return data.token; -} - -function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { - const url = new URL(gitHubUrl); - return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`; -} diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index dc06a22fa0..6f7dc5871e 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -13,8 +13,8 @@ the dominant way dbt is orchestrated today. - **In**: run an unmodified dbt project synced into Windmill, one Windmill job per invocation, live per-model observability, dbt models as first-class assets in the existing asset graph. -- **Out**: one Windmill job per dbt model, `state:modified` / slim CI, - `dbt docs` hosting, semantic layer, dbt platform integration. +- **Out**: one Windmill job per dbt model, slim CI orchestration, `dbt docs` + hosting, semantic layer, dbt platform integration. - **CE**: the runtime, the manifest ingest, the asset graph and every piece of UI ship in CE, as do all adapters except two. Only the `mssql` and `oracle` adapters are EE, mirroring the native `ScriptLang` boundary (decision 21). @@ -35,8 +35,8 @@ the dominant way dbt is orchestrated today. | 10 | Private repo auth | Not applicable: the project is synced, not fetched | | 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | -| 13 | Manifest storage | Sidecar table for nodes/edges. Full manifest **not** stored — see below | -| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below | +| 13 | Manifest storage | Sidecar table for nodes/edges; the whole manifest is kept once per environment, for deferral — see below | +| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** and real column schemas come from the engine's parquet index, opt-in per project — see below | | 15 | Node rendering | Asset nodes per model plus one runnable node for the script | | 16 | Progress | Live, from the JSON event stream | | 17 | Test failures | Honor dbt's own `severity` | @@ -47,6 +47,8 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | +| 26 | Deferral | A durable state per environment, published by the runs whose relations are the script's; `defer` is a per-run toggle. See below | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -157,11 +159,15 @@ build and an enterprise build whose key did not verify. workspace warehouse's NAME, so two scripts running against the same warehouse agree on identity. -The SCHEME names the producer, because dbt is the only thing that creates one of -these: no other language derives warehouse relations, `// materialize` takes -DuckLake targets only, and a dbt run does not dispatch. Calling the kind -something generic promised a parity with native Snowflake and BigQuery scripts -that does not exist. +The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what +put warehouse relations in the asset graph and is what derives them from a +project; no other language *infers* one, and calling the kind something generic +promised a parity with native Snowflake and BigQuery scripts that does not exist. +A script can nonetheless DECLARE that it writes one — `// materialize manual +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -287,6 +293,32 @@ dbt hands the driver — it is written beside `profiles.yml` and pointed at by `sslrootcert`, as it is for a translated postgres resource. `profile.schema` and `threads` from the descriptor override their block keys rather than joining them. +**A `snowflake_oauth` warehouse behaves like a key-pair `snowflake` one**, +although its credential is an access token that lasts ten minutes: + +- The token goes under the `authenticator` each engine reads as an access + token: `oauth` on dbt-core 1.x, `jwt` on the Rust engines. Those read `oauth` + as the refresh-token flow and refuse a profile without client credentials, + while dbt-snowflake has `jwt` only from 1.9 and the 1.x engine can resolve 1.8. +- Every dbt process that logs in re-resolves the warehouse first + (`PreparedProject::refresh_profile`), and resolving an expired OAuth token + refreshes it. So the build, the `after_all` tests, a node retry and the + column-lineage pass each start with a live token, unless resolving fails: the + process then keeps the profile it already has, token included. +- Run identity masks credentials (`RenderedProfile::identity`), so a token + refreshed between a failure and its retry still matches. +- The OAuth connect flow asks for `database`, `warehouse`, `role` and `schema` + (`resource_fields` in `oauth_connect.json`). No token response carries them, + and dbt needs a database. + +One gap stays: a login after the token its dbt process started with has expired +fails. That is a thread whose first connection opens late in a long process, or +a process's first login when the token it was handed had only seconds left. +Refreshing tokens ahead of expiry would narrow it, but a token's lifetime is not +stored, so no margin fits every provider. Closing it would mean handing dbt the +refresh token and the instance's client secret, which any model can read on +dbt-core 1.x. + Three things follow, and they are the reason for the rule rather than consequences to work around. @@ -429,7 +461,9 @@ Two things make that safe rather than a widening: model set and its relations are already visible to them. - **`raw_code` is gated separately**, on an `EXISTS` against `script` in the authed transaction. The body of a model is the project's source code and stays - behind access to the project, whatever the shape query resolved. + behind access to the project, whatever the shape query resolved. `column_schema` + and the column trace behind `/jobs/dbt_column_lineage/{id}` take the same gate, + for the same reason: both are the shape of what the author wrote. The path and hash coming from the job row rather than the query also means a caller cannot pin one project's version while naming another's run. @@ -648,10 +682,13 @@ Two consequences worth knowing: dropped would be filtered out of its own run's graph. The pinned version's nodes are the scope instead. -## No cascade from dbt, and no pipeline membership +## No cascade *from* dbt, and no pipeline membership A finished dbt run does not trigger anything. Its models are recorded, drawn and -tracked; they do not fan out. +tracked; they do not fan out. The opposite direction does: a script that declares +`// materialize manual dbt:////` is an ordinary producer +of that relation, and its completion wakes `# on dbt://` subscribers +through the same fan-out every other asset kind uses. A dbt script is also not a pipeline member (`in_pipeline` is forced false for `ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like @@ -663,22 +700,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what puts a native script reading one of them on the same node, and it is independent of pipeline membership. -dbt already orders its own DAG, so a cascade would only ever add one thing: -waking a Windmill script that reads a mart. That edge is real but narrow, and -only half of it exists — nothing outside dbt can declare a `dbt://` write -(`// materialize` accepts DuckLake targets only), so the reverse direction, an -ingestion script waking a dbt project, cannot be expressed at all. - -Against that, dispatching correctly from dbt is not cheap. A run's `select` can -build any subset of the project, so the deploy-time write set is not what ran; -using it wakes consumers of relations the run never touched, and narrowing it -needs a per-job record of what was built, which the per-relation state table -cannot supply (it keeps one row per relation, stamped with the last writer). +dbt already orders its own DAG, so a cascade out of a run would only ever add one +thing: waking a Windmill script that reads a mart. That edge is real but narrow, +and dispatching it correctly is not cheap. A run's `select` can build any subset +of the project, so the deploy-time write set is not what ran; using it wakes +consumers of relations the run never touched, and narrowing it needs a per-job +record of what was built, which the per-relation state table cannot supply (it +keeps one row per relation, stamped with the last writer). So dbt materializes and reports, and `asset_dispatch` returns early for -`ScriptLang::Dbt`. A `# on dbt://` subscription is refused outright at -deploy rather than accepted and left dormant — an edge drawn on the canvas that -can never fire is worse than an error saying so. +`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should +notify — that decision is the work, not the plumbing. + +### Declaring the write, and which subscriptions are refused + +`// materialize manual dbt:////` is how an ingestion +script says it writes a warehouse relation. `manual` is not a mode but the only +mode: nothing generates warehouse DDL, so the script issues its own write and +Windmill records the outcome — the same `materialized_partition` row a DuckLake +target lands, so the relation carries a last writer on the run page and the graph. +It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path +(`record_declared_warehouse_write`) rather than in an executor, for the same +reason. Identity is unchanged — the physical relation — so the ingestion script +and the dbt model reading it are one node, and a `source` declared on the relation +puts the whole thing on one lineage. `// data_test` is refused beside it: those +checks are probes the DuckDB executor splices around a managed write, so on a +warehouse relation — which the script writes itself, from any language — nothing +would run them, and a declarer would deploy green with its assertions silently +skipped. Assert on the relation with a dbt test in the project that reads it. +The `` segment is resolved at +deploy for the same reason a descriptor's `profile.warehouse` is: a name no +warehouse answers to is not a namespace, it strands the write on a node nothing +else reaches. + +Known boundary, shared with every other runtime pipeline annotation: the record +is written from the normal execution path, and recording and cascading are decided +separately, so the routes off it differ. + +* A **dedicated worker** never enters that path — it bypasses the record exactly + as it bypasses `// partitioned` resolution — while its job is still a top-level + `Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It + cascades and records nothing, leaving the relation with no last writer. +* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which + `is_eligible_kind` rejects. Neither record nor cascade. +* A **flow step running a deployed script** enters the path as a `Script` job, so + it records — and carries a `flow_step_id`, so it never cascades. +* A **flow step with an inline body** is a `FlowScript` job, which the recording + guard excludes along with previews: neither. + +Fixing the recording half is one change for every runtime pipeline annotation, +not this one. + +A `# on dbt://` subscription is held to the same relation a producer +is — a whole `//` under a configured warehouse, checked +by the validator the `// materialize` target goes through, since two spellings of +that rule would refuse and accept the same string. Beyond that it is refused in +exactly one shape: when every script that writes that relation is a dbt one. Nothing +produces it yet is NOT that shape — a subscriber may be deployed before its +producer, as for every other asset kind, and refusing there would break +deploy-order-independent syncs. A dbt script may neither subscribe nor declare a +`// materialize`: its graph ingest republishes that path's trigger and asset rows +wholesale, so either annotation would deploy something the dependency job then +silently removes — while the declared write would still stamp the relation on +every run. + +The producer set is read as it stands committed, minus the deploying script's own +rows — those describe the version being replaced, so a script dropping its +`// materialize` while adding a subscription would otherwise count itself as the +producer that wakes it, which it could not be anyway (the dispatcher skips +self-loops). + +What that leaves is a subscription accepted while it was live and later orphaned. +A dbt project that claims the relation afterwards names those edges in its own log +rather than leaving them silently dormant — the same "an edge that can never fire +is worse than saying so" the refusal is for, at the other point where it is +knowable. Both points that publish ownership warn: the deploy, and a run whose +static descriptor found its profile moved. An agent run publishes none — it is +forced to per-run models, so it stores a job-pinned snapshot and leaves workspace +ownership with the deployed graph — so it cannot orphan a subscription either. + +Two orphanings are reported nowhere, and both are accepted rather than overlooked. +A native producer that drops its `// materialize` and leaves dbt alone on the +relation: the deploy that causes it does not touch the subscriber. And the +interleaving where a dbt ingest commits between a subscriber's producer check and +its own commit — the check sees no producer and accepts, the ingest's warning +query sees no trigger and says nothing. Closing the second means a per-relation +lock shared by the deploy path and the ingest, and the ingest takes +`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation +locks first inverts that order into a deadlock across two subsystems — a worse +failure than the cosmetic edge it would prevent. Both are bounded the same way: +the next deploy of that project warns, and the canvas is where they show +meanwhile. A plain READ still renders the consumer beside the model, which is what makes the lineage one graph — but it is written in the script's own code, not in a @@ -687,8 +801,8 @@ comment: the body parsers resolve an asset URI from a string literal Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the languages with a body-asset parser; the native warehouse ones (snowflake, bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart -they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a -selective run should notify — that decision is the work, not the plumbing. +they consume joins the graph only once that inference exists — while a relation +one of them WRITES joins it now, through the annotation. ## Live per-model progress, and why only dbt-core 1.x has it @@ -857,6 +971,8 @@ profile: select: ["tag:nightly+"] exclude: [] test_behavior: build # build | after_all | none +column_lineage: false # opt in to the static-analysis pass that + # produces column-level lineage (decision 14) vars: # typed: numbers/bools/lists keep their type, run_date: "{{ run_date }}" # and string leaves take job arguments strict: false @@ -1143,19 +1259,513 @@ block, since that is what `dbt_run_state` saves and `invocation_args` publishes. without failing. Overriding this would make the same project behave differently on Windmill than locally, breaking the core promise. +## Durable state per environment, and what defers to it + +`dbt --defer --state ` resolves a `ref()` the run does not build to the +relation the manifest in `` names, instead of to the schema this run writes +into. That is what lets one model be rebuilt into a scratch schema without +rebuilding everything above it, and it needs a manifest of the environment the +project actually lives in. + +Nothing that already existed could supply one. `dbt_run_state` answers a +different question — it holds the LAST run whatever its outcome, keyed by the +principal, so `dbt retry` can resume its failures — and the worker-local +generations behind it are a cache: the next run of a project usually lands on a +worker holding neither artifact. So the state is its own table, +`dbt_environment_state`, one row per (workspace, script path, environment), +holding `manifest.json` and `run_results.json` from the last SUCCESSFUL run. +Success is half of the contract: a relation a later run defers to has to exist. + +### The environment is the warehouse, the target and where they resolve to + +The workspace warehouse's name, the target dbt actually runs, and the database +and schema that target resolves to — the pair `relation_root` reports to the +graph's drift check. Each component is length-prefixed rather than joined on a +separator — `|||`, each written `:`, +so `main`/`prod`/`analytics`/`dbt_wh_defer` is stored as +`4:main|4:prod|9:analytics|12:dbt_wh_defer`. A target name and a schema are both +the user's own strings, so `prod|analytics` + `scratch` and `prod` + +`analytics|scratch` would otherwise be one key, and a profile moving between them +would read as the same environment rather than as one nothing has published. What +a message names is spelled out instead, never the encoded key. + +The target is the EFFECTIVE one, not the descriptor's `profile.target`: a +descriptor naming none inherits the workspace warehouse's, or the default in the +project's own `profiles.yml`, so reading the descriptor's would file every +inherited target under one empty name — and a `target.name` macro decides where a +model is built. + +The last two are in the key because deferring is resolving a relation NAME. A +warehouse repointed at another database, or a `profile.schema` moved by a +redeploy, keeps the first two while putting every relation somewhere else, and a +manifest is a list of relation names — there is no other way to notice. Keyed on +the first two alone, such a move would hand the next deferring run the names of +relations that are no longer there. Keyed on all four, it reads as an +environment nothing has published yet, which is what it is. + +What the key deliberately does NOT carry is the resolved connection. That is the +`profile_digest` a retry is held to. It masks credentials, but it still moves +with changes that move no relation, like another Snowflake warehouse or role; a +warehouse pointing somewhere else entirely is decision 11's accepted limitation, +spelled the same way here as everywhere else. + +Today one script has one environment, because a descriptor fixes both the +warehouse and the target and a run cannot override either. The key is what makes +the *later* item — fork and preview environments — an addition rather than a +migration, and what makes a profile move detectable now. + +### Which runs publish it + +A successful `build` that did not itself defer, and whose graph becomes what the +script owns (`GraphRefresh::publishes_ownership`) — the same condition as the +graph's and the same reason: an invocation that scoped its own model set — a +`vars` or `select` override, or a descriptor dynamic by construction — describes +where the CALLER put those relations, not where this project's models live. +Publishing it would point every later deferral at one caller's scratch schema. + +**A run that deferred never publishes, whatever narrowed it**, and that is a +separate condition rather than a consequence of the first. A deferring run built +some of the relations its manifest names and resolved the rest out of the state +it read, so recording that manifest would claim relations nothing built — and a +model renamed since would be recorded under a name only a full build creates, +breaking every later deferral until one repairs it. `publishes_ownership` cannot +see this: it reads the caller's overrides, and a descriptor that already narrows +`select` needs none. + +A `retry` publishes nothing. Its `run_results.json` names only the nodes it +redid, so the environment would come to claim a run of a handful of models. The +environment's state is therefore the last full successful build, exactly as dbt +Cloud's "last successful run" is, and a run recovered by a retry leaves it at +the previous one. + +The AUTOMATIC in-job node retry is the same artifact under a different name: a +build it recovers is a successful build, but the `run_results.json` on disk is +the retry's. Such a run publishes the manifest **without** results, rather than +with a set describing some other slice of the build — the manifest is a function +of the project rather than of what ran, so deferral is unaffected. A `result:` +selector is the one thing left with nothing to read, and it is refused by name +against such a publication rather than passed to dbt (see "Selectors that read +the state" below). + +Under `test_behavior: after_all` the stored `run_results.json` is the test +phase's, because that is what the second invocation leaves in the target +directory — the same artifact a local `dbt run && dbt test` leaves behind. + +**What that condition means for what the artifacts may carry**, and why this +table is keyed by environment where `dbt_run_state` is keyed by principal. dbt +records the invocation's flags into `run_results.json`, and Windmill resolves +`$var:` / `$res:` references before dbt sees them — which is exactly why the +retry state is per-principal, so one caller's resolved `select` and `vars` are +not restorable by the next. Here they cannot be one caller's: a publishing run +added nothing of its own, and a descriptor that interpolates a `{{ }}` +placeholder into `vars` never publishes at all, so what is recorded is the +descriptor's own arguments — the script's content, which anyone entitled to run +it may already read. Widen the publish condition and that stops being true. + +### Where the blob goes + +`run_results.json` is small; `manifest.json` is not, and grows with the project +(535 KB on a two-model fixture). Each takes the same two homes: inline in the row +under `DBT_STATE_INLINE_MAX_BYTES` (8 MiB), and the INSTANCE's object storage +above it, with the row keeping the key. Inline is what makes the feature work on +an instance that has configured no storage at all; the ceiling is what stops one +project's manifest from becoming a multi-megabyte row rewritten by every run. A +project past the ceiling with no storage configured is told so, in the job log, +naming the setting and the variable — the run itself still succeeds, since +losing the state costs the next deferral rather than the build that just ran. + +**The instance store, not the workspace's**, which is where every other internal +worker artifact already lives (bun bundles, python wheels, job logs, the global +cache). The workspace bucket is the one members read and write through +`job_helpers/*` and `wmill.write_s3_file` with a caller-supplied key, and only +`volumes/` is reserved there — so a manifest under it is one any member could +replace, and the next deferring run would hand dbt an attacker-chosen +`defer_relation` for every unbuilt `ref()` while holding the script's warehouse +credentials. Its compiled SQL would be readable there too, for a project the +reader may have no access to. The consequence to know: a project past the ceiling +needs the instance store configured, which is an EE feature, so on CE the ceiling +is the limit and `DBT_STATE_INLINE_MAX_BYTES` is how it moves. + +Each publication writes its OWN keys +(`wmill_dbt_state///./`) +and the row switches to them in one statement, so an upload never overwrites an +artifact the committed row still names: a run that fails between its two uploads, +or between them and its row, leaves the state pointing at the pair it already +had. The objects the commit displaced are dropped afterwards, never before, since +a reader that has already read the row is about to fetch them; a reader that +loses that race re-reads for as long as the row keeps MOVING, rather than +reporting a state that is there. A reader takes no lock, so successive +publications can each overtake one; an unmoved row whose objects are gone is the +error that means what it says, and a bound on the re-reads is the other, for a +project republishing faster than a run can read. What a publication uploaded and then could not commit is dropped on the +way out — except after a commit that REPORTED an error, where what was lost may +be only the acknowledgement: dropping then would leave a committed row naming +objects that are gone, so an orphan is the cheaper side to take. + +The path and the environment are only a prefix of that key. The row is what says +where an artifact is, which is why state can travel with a renamed script and go +on naming objects under the old path's digest. The rest of the key is the job and +a per-EXECUTION nonce — zombie recovery re-runs a job under its own id, so keyed +on that alone a second attempt would overwrite the objects the first attempt's +committed row still names, then read those keys back as displaced and drop them. + +Publishers of one environment serialize on `pg_advisory_xact_lock`, so only one +of them settles the row and the objects it displaces at a time — an advisory lock +rather than the row's, because the first publish of an environment has no row to +lock and is exactly when two runs of a newly deployed script are most likely to +race. + +### Retention + +None, deliberately, and this is where it differs from the graph tables next +door. Those are pruned by age by the dbt runs themselves because their reader is +a transient run page. This one holds a single row per script per environment, +replaced in place, so it does not grow with runs — and its reader is every later +run of that script, so a project that runs monthly must still find last month's +state. It goes with the script instead: a path no live dbt version occupies any +more clears it, alongside `dbt_run_state` (`clear_dbt_script_state`, +`clear_dbt_script_state_if_path_retired`). + +The write carries a guard of its own, and it names the VERSION rather than the +path: the live dbt script there must be the one this job ran, or a later version +of it (`hash = $n OR $n = ANY(parent_hashes)`). "Some live dbt script is here" — +which is what the retry state settles for — is also satisfied by a script created +at a path this one was renamed away from, and this job's manifest would then +become that project's deferral state. A preview names no version and so publishes +nothing, which is right for a run of content that was never deployed. + +The job's KIND is checked beside it, because a preview carries a caller-supplied +`script_hash` into `runnable_id` (`run_preview_script`): the version alone would +let anyone who may run a job publish arbitrary content as a deployed script's +state. A flow or app step naming a deployed dbt script by path is an ordinary +`script` job carrying that script's own hash, so it publishes like any other run; +only INLINE flow code is a `FlowScript`, and that has no deployed version to +publish for. + +That guard HOLDS the script row (`FOR SHARE`) for the rest of the publication, so +a rename, archive or delete of the path either waits for it or is seen by it. +Read unlocked, it leaves a window where the lifecycle clear finds no row to take, +finishes, and the publication then commits state at a path a new script goes on +to occupy. The script row is taken before the sidecar, which is the order every +other dbt writer takes and what keeps the two off a deadlock. + +An artifact too large for its row is left in the store when the row is cleared, +as a deleted script leaves its bundle: reaching it from the delete would mean an +object-store client in `windmill-common` and a delete that has to land after the +caller's transaction commits, for one object per environment of a script that is +gone. + +### Asking for it + +`defer` is a field on the `build` command block, defaulting to the descriptor's +own `defer:`. A per-run toggle rather than a descriptor-only setting, because the +run that publishes an environment's state and the run that defers to it are two +invocations of ONE script (decision 6: N scripts means N projects): a project +that could only defer by descriptor could never populate the state it reads. + +A project whose profile selects its schema or database with a TEMPLATE — either +delimiter, since dbt renders `{% … %}` blocks as well as `{{ … }}` — is refused a +deferral outright, and publishes no state either: dbt renders those and Windmill +does not, so two renderings resolve to one `relation_root`, and a +deferral after the value changed would resolve every unbuilt `ref()` through the +previous location's manifest. Both sides, because a published template would sit +under a key a literal profile shares, and de-templating later would make that +stale manifest readable as the new location's. It covers a project-owned +`profiles.yml`, a `dbt_profile` resource — one block of the user's own file, +copied through unchanged — and a `profile.schema` written as given. Plainly +absent is different: that is the adapter's default, which does not move. + +A run that asks to defer with nothing published is refused, naming the +environment and the runs that cannot publish one. The alternative — running +without deferral — fails deep inside dbt with a relation-not-found the caller has +no way to connect back to a missing state. An agent worker is refused the same +way and for a reason it can act on: it reaches the database only through the API, +which does not expose this table. + +A `show` defers too, and every engine takes the flags on it. It compiles the +model it previews, so a model whose upstream this environment built and this run +did not is exactly the case a deferral exists for. So does the `dbt ls` that +resolves what a run's selection owns, without which a `result:` selector — which +reads `run_results.json` out of the state directory, and which `select` passes to +dbt verbatim — would fail before the build that would have honoured it. + +The result carries `deferred_to`, the run whose state was used. Without it what +a deferring run built against is unrecoverable, since the next successful run of +that environment replaces the state. + +### Selectors that read the state, and why they are refused rather than passed + +`--state` also feeds dbt's own selector methods, so publishing the state is what +makes `state:modified+`, `state:new` and `result:error+` resolve at all. Only a +deferring run is handed the directory, so a `state:` or `result:` method in +`select` or `exclude` without `defer` is refused before dbt starts. + +Refused, rather than left to dbt, because the engines disagree about it and two +of the three disagree silently. Given a state selector and no `--state`, +dbt-core 1.x raises (`Got a state selector method, but no comparison manifest`, +exit 2), but dbt-sa-cli 2.x and fusion read a MISSING state as an EMPTY one and +exit 0: `state:modified` then selects nothing and the run reports success having +built nothing, while `state:new` selects everything, because against an empty +state every node is new. A scheduled run that quietly stops doing work, or +quietly rebuilds the project, is the failure this state exists to prevent. + +From the DESCRIPTOR they are refused whether or not the run defers, and the +message says so. That selection is also what decides which nodes the script owns, +and the deploy resolves it before any run exists, with no state to compare +against. "Whatever changed last" is not an ownership answer. They describe one +run, so they belong in a run's own `select`. + +`source_status:` is refused under any setting: it compares `sources.json`, which +`dbt source freshness` writes and no run publishes here, so there is nothing to +compare against even while deferring. + +Two more refusals follow from the same argument, that a selector with nothing to +read must say so rather than resolve to a silent answer: + +- A `result:` method while deferring to a state that carries **no** + `run_results.json`. Publishing that is deliberate — a build recovered by + automatic node retry stores the manifest alone, its results describing the + retried nodes rather than the build ("Which runs publish it") — so `defer` + being on is not enough to know the file is there. Answerable only once the + state is loaded, so it is checked right after, naming the run that published. +- Any of them on a `parse`. A parse resolves a selection to store the graph and + never defers, so `defer` would not hand it a state at any setting, and the + remedy the other refusal offers would lead nowhere. It says that instead. + +Matching nothing is then an ordinary outcome for these methods, and for no +others. `state:modified+` selects the empty set exactly when nothing changed +since the published state, which is the answer a CI run wants, so a selection +naming a `state:` or `result:` method may resolve to no nodes. Such a run scoped +its own selection, so what it stores is a snapshot of its own and never what the +script owns, and nothing is un-wired by the empty set. + +The exemption is by METHOD, not by who chose the selection. Exempting every +caller-chosen one would take a misspelled model name, which resolves to nothing +just as surely, and report it as a build that did its work. An ordinary selection +matching nothing stays refused, from a run as from the descriptor — from the +descriptor because that one also decides ownership. + +Only what `select` and `exclude` spell directly. A method reached through a +`selectors.yml` definition is named nowhere the worker reads, and dbt's own +behaviour — including the silent one — is what stands there. + +### `--state` is also a retry's own argument, and that is a trap + +`dbt retry` reads the run it RESUMES from `--state`. Handed the deferral's +directory it resumes the successful run stored there, finds nothing failed, and +reports a green retry having rebuilt nothing — silently, on dbt-core 1.x, which +warns and exits 0. + +dbt-core 1.x has `--defer-state`, the deferral-only half of the pair, so a retry +there passes that and leaves `--state` alone. The Rust engines do not have it, +and a run that deferred is refused a retry on them, before the build: the +alternative is rebuilding the failed nodes with every `ref()` resolving into the +schema this run writes into, which for the narrowed run a deferral exists to +serve means writing them somewhere they do not belong. The automatic in-job node +retry is dropped for the same reason and says so in the log. + +The state directory is passed RELATIVE (`wm_dbt_state`, beside `wm_target` in the +job directory). dbt records the invocation's flags into `run_results.json` and a +later `dbt retry` restores them, so an absolute path would name the job directory +of the run being resumed, which is gone by then. Relative, it resolves against +the project root — whichever job directory the retry landed in. + +Three engine facts found while wiring this up, all worth knowing before filing a +bug against the feature. `dbt retry` on dbt-core 2.x restores **neither** the +resumed invocation's `--vars` nor its deferral: it re-parses with the current +(empty) ones, so a retry of a run that overrode `vars` rebuilds into the +descriptor's schema rather than the run's. That is independent of deferral and +predates it; the refusal above stops the deferring case from being the way it is +discovered. `dbt show` on either Rust engine prints a bare JSON array where +dbt-core frames it as `{"node": …, "show": […]}`, which `run_show` is written +against — so a preview there fails to parse whether or not it defers, and the +deferral itself resolves correctly under it. And neither Rust engine reached +dbt's own service-backed State (`--manage-state`) on any run measured here, so no +flag is passed to disable it. + +Because `select` reaches dbt verbatim, a deferring run also has a `--state` +directory for `result:` selectors, which is why `run_results.json` is stored +beside the manifest rather than the manifest alone. + ## Two decisions the implementation narrowed -**Decision 13 — no S3 copy of the manifest.** The sidecar holds every field the -graph renders; nothing reads a stored `manifest.json`, so writing one to S3 -would be an unread copy of data that is already reproducible by redeploying (or, -for a dynamic descriptor, by the next run). Worth adding the day something needs the -parts the sidecar drops — compiled SQL, macro definitions — and not before. +**Decision 13 — the manifest is stored once per environment, not per version.** +The sidecar holds every field the graph renders, so a copy of `manifest.json` +bought the graph nothing: it is reproducible by redeploying, or for a dynamic +descriptor by the next run. Deferral is the reader that changed that — it +resolves an unbuilt `ref()` through a manifest, and one on worker-local disk +answers for a machine's history rather than for the environment. So exactly one +manifest is kept per (script, environment), replaced by each successful run, +rather than one per version (see "Durable state per environment" above). -**Decision 14 — column lineage is not available.** The decision assumed -`manifest.json` carries column-to-column edges; it does not, in either core -engine. What it does carry is declared column *descriptions*, which are -ingested. Real column lineage would need Fusion (which does static analysis) or -a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. +**Decision 14 — column lineage comes from the parquet index, not the manifest.** +`manifest.json` carries no column-to-column edges, in any engine, and its +`columns` are the ones an author declared in `schema.yml`. Both halves exist in a +different artifact: `dbt compile --static-analysis strict --write-index` writes +`target/index/`, and two of its tables are `dbt.column_lineage.parquet` +(`from_node_unique_id`, `from_column_name`, `to_node_unique_id`, +`to_column_name`, `lineage_kind`) and `dbt.node_columns.parquet` (every column of +every node, with its declared type, its inferred type and its description). + +Three measured properties decide the shape of the ingest. + +**Strict analysis is a stricter dialect.** `select no_such_column from +ref(...)` is `UnresolvedIdentifier (dbt0227)` and exit 1 under `strict`, and +compiles fine under `baseline` (the default). So this is a separate `dbt compile` +with its own `--target-path`, never a flag on the build, and it is opt-in per +project: `column_lineage: true` in the descriptor. Off, nothing changes. On, a +project that cannot be analyzed keeps exactly the graph it had. + +The pass is best-effort about everything that is ITS: a wrong engine, a rejected +analysis, a missing or unreadable artifact, an over-long output and outrunning its +own time budget all degrade to partial lineage or none, plus a line in the job +log saying which. It is not best-effort about the JOB: a cancellation or the job's +own deadline fail it, because swallowing those would let a run that blew its +timeout inside an optional annotation publish a graph and report success. That +split is why the two halves have separate error contracts — the compile owns the +job's semantics and may `Err`; nothing the artifact does or fails to do is a +reason to fail a job, so an absent, unreadable or partial index is a value. The +decode still runs under the job poller, which both heartbeats through it and +ends it if the job is cancelled or completed meanwhile: the job reaching in, not +the artifact reaching out. The +budget is half the job's remaining wall clock, spent on the compile alone, so the +build that follows cannot be starved by it. + +**A failed pass still writes the index**, holding every edge of the models that +did analyze, so the artifact is read whatever the exit status and partial lineage +is a normal outcome. An unreachable *source* is milder still: `RemoteError +(dbt1014)` downgrades that model to `static_analysis: off` and the compile +succeeds. (Strict analysis queries the warehouse catalog for source schemas; a +`ref()`ed model is inferred statically and needs no built table.) + +**The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 — the version +`DBT_CORE_2X_VERSION` pins — accepts `--write-index` and `--write-lineage`, and +its own `views.sql` declares views over both tables, but it writes neither +parquet; only Fusion does today. The ADAPTER decides too: an experimental one +(postgres under `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`) turns static analysis off and +says so only in a warning on an otherwise successful compile. The gate is +therefore "the engine has the flag" (everything but 1.x, whose Python CLI has no +such option) plus "the file appeared", so a later release picking the feature up +needs no change here — and the job log carries the engine's own stderr whenever +no index appears, since without it "no column lineage" has no explanation. + +`lineage_kind` is stored as TEXT, not an enum. Three values exist — `copy` +(passthrough), `mod` (transformed) and `scan` (the column was read to produce the +ROW rather than the value: a join key, a `where` predicate, a `group by`) — and +the engine's own reader maps those three and passes anything else through, so the +set is the engine's to extend. All three are stored, and `copy`/`mod` are kept +first when the bound bites: a `scan` edge reaches every output column of its +model, so it is most of what a project's index holds and would draw as a complete +bipartite graph. Keeping it in the table anyway is what lets a later "show +indirect" view ask for it without every project being redeployed. + +Storage mirrors `dbt_edge` exactly: `dbt_column_edge`, keyed by (path, version, +job) with the same composite foreign key to `script`, so a version's column +lineage dies with the version and a run's snapshot with the sweep. + +**A table of its own, not `dbt_edge.column_lineage` JSONB.** Hanging the links on +the `ref()` edge they sit beneath would inherit its clone, prune, clear and +cascade paths for free, and it does not work: a model reading `{{ this }}` gets +column lineage from itself to itself, and `parent_map` has no self-loop, because +a model does not `ref()` itself. Those pairs have no `dbt_edge` row to attach to. +The loss is not hypothetical — an incremental that selects from `{{ this }}` +(`coalesce(p.dbl, s.dbl)`, `p.up as prev_up`) yields `up → prev_up` with kind +`copy`, a drawn edge meaning "this column carries the previous run's value". +Inventing self-loop `dbt_edge` rows to hold it is not an option either: that +table is `ref()` lineage. The typed column list lands in +`dbt_node.column_schema`, beside `columns` rather than merged into it — +`columns` stays what the author *declared*. + +Two things are user-visible. `column_schema` — every column of a relation, typed +and in the order the model emits them — rides the asset graph the details pane +already fetches, and replaces a panel that could only list the columns an author +happened to document. The edges are served by an endpoint of their own, +`assets/column_lineage`, which the pane asks for the selection it is drawing. + +Both are gated on being able to read the producing project, like the model's SQL: +a column-level view is the shape of what the author wrote, one level finer than +the `ref()` graph, which is ungated only because it draws relations the caller +already sees. A share-link viewer entitled to a dbt run therefore gets its +relations and `ref()` edges, and neither the SQL nor the columns. + +**One request per selection, and the gate re-decided per project.** The endpoint +takes every relation the view has reached and answers their union, because one +selection reaches several — a script's output column can derive from columns of +several models. Holding partial answers between selections instead was tried and +is what a client cache is: it produced a wrong premise for a relation two projects +describe, then staleness on redeploy, then a lost retry. + +The answer is the connected component around those relations, and that component +does not stop at the project that owns them: a relation one project produces is +another's source, so the walk resolves owners, reads their edges, walks, and +repeats for the relations that walk newly reached. Resolving once — for the +relations asked about — stops the trace at the first project boundary. The +security half is that a project reached this way is a project the caller may not +be entitled to, so the scope filter and the project's visibility are re-applied to +every project the expansion discovers, not decided once for the first owner set. + +A PINNED answer is the exception and needs none of it, whether it names a job or +a deployed version: the pin says which stored graph is on screen, and another +project's live graph is not part of it, so it answers for that one project. The +dbt editor pins by version on every selection and the run page pins by job; the +pipeline page pins nothing, and is where the walk crosses projects. + +**The component is bounded, and says when it was cut.** The renderer draws a box +per column, so a component past a few thousand edges is unreadable however it is +served — a synthetic 3000-model project whose models share a column returns 58k +direct edges and 7.3MB. The walk is breadth-first from the asked-for relations and +stops at 5000 edges, so what survives is the part nearest the selection rather +than an arbitrary slice, and `truncated` says so: a trace that stops short is +otherwise indistinguishable from one that ends. + +`truncated` is set only where a project this caller may read is left unread, or +where the walk itself was cut. Neither the expansion's size budget nor a relation +whose owners were never asked about is evidence on its own: a big project's small +component is answered whole, and the relations still waiting to be asked about at +the end are almost always owned by the project already in hand. The order the +rounds run in is what makes that decidable — the owners query comes before the +budget check, so the stop happens with a named unread project rather than a +suspicion of one. + +The FETCH is not bounded the way the answer is: a project's edges arrive whole, +because the walk is what decides which of them are in the component, and a +`LIMIT` would cut a set that need not contain the asked-for relation at all. The +seeds' own projects are therefore read whatever their size — reading them is the +answer. What bounds a single fetch is the ingest's own cap per version; the +budget bounds only the expansion on top of it. + +The walk is in Rust rather than a recursive CTE. `EXPLAIN ANALYZE` on that same +project measured 1243ms against 59ms for the query alone: a CTE has no index to +walk, so the recursive term rescans the doubled edge set once per level (11.7M +rows), while the same walk over a map is microseconds. + +The two halves of a trace — dbt's and the pipeline's — meet at shared node ids. A +DuckDB script's `// column x <- dbt://wh/s/model.col` mints the same +`(dbt, path, column)` node dbt's own lineage does, so the producer graph the asset +graph already carries and the dbt graph are MERGED rather than chosen between, and +a trace crosses that boundary in either direction. What the browser cannot close +in one request is a relation the server discovers whose columns are consumed by a +script that writes into a third project: the producer half of that hop is the +canvas's, not the server's, and the seeds were computed before the answer arrived. + +**The analysis pass takes the build's own `--full-refresh`.** `is_incremental()` +branches on it, so an incremental model reading `{{ this }}` compiles its +self-join — and any `ref()` inside that branch — only when the flag is absent. A +pass that used the descriptor's default while the run overrode it would store +lineage for SQL that run never executed. For the same reason an invocation that +overrides the flag counts as `per_run_models`: its graph is its own, keyed to the +job, rather than standing as the version's. + +That flag is not the whole of it, and the rest is a property rather than a bug to +fix. `is_incremental()` is also false when the target table does not exist, so an +incremental model has **two shapes and one ingest holds one of them**: a deploy +before the first build compiles the cold shape, and the same project deployed +again once its tables exist compiles the incremental one. A static descriptor +re-ingests on neither runs nor time, so what is stored stays whatever the compile +in front of it saw. dbt has no mode that emits both, and re-analyzing per run +would buy a second `dbt compile` on every build to keep a graph nobody asked to +refresh. The contract is therefore the honest one: a version's graph describes +the compile that produced it, and a run that re-ingests describes its own run. ## Concept mapping @@ -1167,7 +1777,9 @@ a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. | `materialized: incremental` | `append` or `merge` (by `unique_key`) | same | | `{% snapshot %}` | `scd2` | same, incl. `_current` handling | | `unique`/`not_null`/`accepted_values`/`relationships` | `data_tests` | exact 1:1 with the four `// data_test` kinds | -| declared column metadata | `columns` on the asset node | descriptions only; see the note below | +| declared column metadata | `columns` on the asset node | descriptions only, from the manifest | +| analyzed column schema | `column_schema` on the asset node | `dbt.node_columns.parquet`, opt-in | +| column-to-column lineage | `dbt_column_edge` rows, drawn as a column trace | `dbt.column_lineage.parquet`, opt-in | | model `tags` | node badge | `tag` | | source freshness | `freshness` | `last_success_at` chip | | `run_results.json` | materialization records | `record_materialization` | @@ -1195,7 +1807,10 @@ render through the existing `RunnableNode.svelte` / `AssetNode.svelte` / on the canvas mid-run. `record_materialization` per model. Profile and select pickers in the editor. Per-model failure triage in the run view. -**Phase 4 (not in this PR).** `--defer` and `state:modified`. Partition and +**Phase 4 (not in this PR).** Slim CI: the fork and preview environments a +deferral would name instead of its own. The selectors themselves are here, since +`state:` and `result:` read the published state like any deferral does; what is +missing is a per-branch environment to compare a CI run against. Partition and backfill integration so `BackfillRangeDialog.svelte` works on dbt models. `wmill dbt import ` reading `DbtDag(...)` kwargs. @@ -1215,18 +1830,33 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: script reading one of the marts gets an edge to it. 6. **Shared node**: a native script that READS a mart renders as a reader of the same node the dbt model writes — one node, not two islands. Declared with a - plain read (`# dbt://`), never `# on`: a `dbt://` subscription is - refused at deploy, because nothing but dbt writes a warehouse relation and a - dbt run does not dispatch (see "no cascade from dbt"). -7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each + plain read (`# dbt://`), never `# on`: a subscription to a relation dbt + alone builds is refused at deploy, since a dbt run does not dispatch. +7. **Declared write**: a native `// materialize manual dbt://` script + and a dbt project reading that relation as a `source` render as one node; a + run of the script records its materialization and wakes a + `# on dbt://` subscriber — a subscription only that producer makes + wakeable, the dbt project reading the relation being no producer of it (see + "no cascade *from* dbt"). +8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each build only the expected subset. -8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph +9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph from the run's own manifest, so a model that placeholder enables appears in the same run that builds it. -9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's +10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's own `profiles.yml` with env-var injection. -10. **Caching**: a second run reuses the cached `dbt_packages/` with no network +11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. +12. **Deferral**: a full run publishes the environment's state; a second run + that builds one downstream model into another schema resolves its unbuilt + `ref()` to the relation the state names, where the same run without `defer` + fails with relation-not-found. +13. **State selectors**: with a state published, `state:modified+` selects + nothing while the project is unchanged and exactly the changed model and its + children after one is edited. Without `defer` it is refused rather than + passed, and a `result:` selector against a state published by a + node-retry-recovered build is refused too, that one carrying no + `run_results.json`. Keep only tests that pin behavior a future change could break. Per AGENTS.md, delete development scaffolding before marking the PR ready. diff --git a/docs/docker-security.md b/docs/docker-security.md index f0a9f23073..5b5196f8b4 100644 --- a/docs/docker-security.md +++ b/docs/docker-security.md @@ -63,3 +63,65 @@ gap, rebuild and republish the `latest` / patch tags: Scan the published images (e.g. Trivy / Defender) after rebuilds to confirm the base-OS finding count stays low. + +# Verifying image signatures, SBOMs and provenance + +Release images are signed and attested at publish time: + +- **cosign keyless signature** on the pushed manifest digest (index and + per-arch manifests), via GitHub OIDC — no long-lived signing key exists + (`.github/actions/sign-attest-image`). +- **SBOMs** are generated at build time (`sbom: true` on the depot build + step) and embedded in the image index as BuildKit attestation manifests — + one SPDX document per platform. They are part of the signed index digest, + so the cosign signature covers them. They are not sent to a transparency + log: SPDX documents for these images run tens of MB, beyond what Rekor or + GitHub attestations accept as payloads. +- **SLSA build provenance** recorded as a GitHub artifact attestation and + pushed to the registry (`actions/attest-build-provenance`). + +## What is covered + +Only images published from a release tag (`v*`) are signed: `windmill`, +`windmill-ee`, `windmill-ee-cuda`, `windmill-slim`, `windmill-ee-slim`, +`windmill-full`, `windmill-ee-full` (`.github/workflows/docker-image.yml`), +`windmill-cli` (`build_cli_image.yml`) and `windmill-extra` +(`publish_extra.yml`). The `:latest` and `:main` tags are repointed on +every `main` push as well as on releases, so they resolve to a signed +digest only until the next `main` build lands — verify a version tag or a +digest, not `:latest`. Development images (`:dev`, branch builds, +`windmill-test`), the dispatch-only RHEL/rpi images and the `caddy-l4` +image are not signed. + +## How to verify + +Signatures are keyless: trust is anchored in the Fulcio certificate identity, +which for these images is the *calling workflow file at a `v*` tag ref* in +this repository. Verify a signature with cosign (v2.x): + +```bash +cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp '^https://github.com/windmill-labs/windmill/\.github/workflows/(docker-image|publish_extra|build_cli_image)\.yml@refs/tags/v' \ + ghcr.io/windmill-labs/windmill: +``` + +Extract the embedded SBOM (per platform; verify the signature first — it +covers the index these documents live in): + +```bash +docker buildx imagetools inspect ghcr.io/windmill-labs/windmill: \ + --format '{{ json .SBOM }}' +``` + +Verify SLSA provenance through GitHub's attestation API: + +```bash +gh attestation verify oci://ghcr.io/windmill-labs/windmill: \ + -R windmill-labs/windmill +``` + +Note for registry housekeeping: cosign stores signatures as extra +`sha256-.sig` tags in the same ghcr package, and the pushed +provenance attestations live there as referrer artifacts — any +tag-retention automation must not prune them. diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 32fff7ca23..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion (EL)"; this section records only what future feature work must not break. -- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and - `manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK +- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected + elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** — diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 5d7f0b2c29..ac64800c36 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,9 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 28 registered actions across fourteen features (`ai_session`, `ai_chat`, -`ai_fix`, `ai_agent`, `ai_agent_eval`, `flow_editor`, `flow_run`, `flow_step`, `run_form`, -`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`). Nearly all of the +It currently carries 49 registered actions across eighteen features (`ai_session`, `ai_chat`, +`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, +`flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, +`usage_meter`, `sso_groups_claim`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/docs/git-repo-viewer-hub-script.md b/docs/git-repo-viewer-hub-script.md index 7bfbe8160f..a1b5ea1482 100644 --- a/docs/git-repo-viewer-hub-script.md +++ b/docs/git-repo-viewer-hub-script.md @@ -1,7 +1,10 @@ # Git repo viewer — hub script -The hub script `clone_repo_and_upload_to_instance_storage` is published from -`windmill-integrations` and pinned in `frontend/src/lib/hubPaths.json` as +The hub script `clone_repo_and_upload_to_instance_storage` +([hub page](https://hub.windmill.dev/scripts/windmill/13968)) is published from +`windmill-integrations` +(`hub/windmill/scripts/action/13968_clone_repo_and_upload_to_instance_storage/script.ts`) +and pinned in `frontend/src/lib/hubPaths.json` as `cloneRepoToS3forGitRepoViewer`. Hub paths are exact version pins, so editing the script means publishing a new version and repointing that entry. @@ -23,6 +26,11 @@ The repo viewer in the Windmill app expects the hub script to: 3. **Write a completion marker** as the very last action of a successful run, so the API and frontend can distinguish a fully-populated S3 directory from a partial / interrupted upload. +4. **Follow symlinks that stay inside the checkout.** Both the git clone and + the archive extraction keep a repository's symlinks as links, and + `Dirent.isFile()` / `isDirectory()` are both false for a link, so a walk + that only checks those drops every linked file and directory from the + viewer. See [Symlinks](#symlinks). The marker file the frontend looks for is `.windmill_clone_complete` at the root of the per-commit directory: @@ -43,28 +51,74 @@ after the walk completes: ```ts const UPLOAD_CONCURRENCY = 16 const CLONE_MARKER_FILE = ".windmill_clone_complete" +const MAX_SYMLINKED_ENTRIES = 20_000 +const MAX_SYMLINKED_BYTES = 512 * 1024 * 1024 async function uploadDirectoryToS3( directoryPath: string, s3BasePath: string, workspace: string, -) { +): Promise { console.log(`Uploading ${directoryPath} -> ${s3BasePath}`) // Walk the directory once, producing a flat list of (localPath, s3Key) pairs. const tasks: { localPath: string; s3Key: string }[] = [] - function walk(dir: string, s3Path: string) { + const root = fs.realpathSync(directoryPath) + // Real paths of the directories being descended through. + const ancestors = new Set() + // What entries reached through a link have cost so far; see Symlinks below. + let symlinkedEntries = 0 + let symlinkedBytes = 0 + let symlinkBudgetSpent = false + function chargeSymlinkBudget(relPath: string, entries: number, bytes: number): boolean { + if (symlinkBudgetSpent) return false + symlinkedEntries += entries + symlinkedBytes += bytes + if (symlinkedEntries <= MAX_SYMLINKED_ENTRIES && symlinkedBytes <= MAX_SYMLINKED_BYTES) { + return true + } + symlinkBudgetSpent = true + console.log( + `Skipping ${relPath} and every symlinked entry after it: symlinks reach more than ` + + `${MAX_SYMLINKED_ENTRIES} entries or ${MAX_SYMLINKED_BYTES / 2 ** 20} MiB` + ) + return false + } + function walk(dir: string, relDir: string, viaLink: boolean) { + ancestors.add(dir) for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name) - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name - if (entry.isDirectory()) { - walk(fullPath, s3Key) - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }) + const relPath = relDir ? `${relDir}/${entry.name}` : entry.name + const linked = viaLink || entry.isSymbolicLink() + if (linked && !chargeSymlinkBudget(relPath, 1, 0)) continue + let localPath = join(dir, entry.name) + if (entry.isSymbolicLink()) { + const link = fs.readlinkSync(localPath) + try { + localPath = fs.realpathSync(localPath) + } catch (e: any) { + console.log(`Skipping symlink ${relPath} -> ${link}: cannot resolve target (${e.code})`) + continue + } + if (localPath !== root && !localPath.startsWith(root + sep)) { + console.log(`Skipping symlink ${relPath} -> ${link}: target is outside the repository`) + continue + } + } + const stat = fs.statSync(localPath) + if (stat.isDirectory() && ancestors.has(localPath)) { + console.log(`Skipping ${relPath}: links back to a directory it is inside`) + continue + } + if (linked && stat.isFile() && !chargeSymlinkBudget(relPath, 0, stat.size)) continue + if (stat.isDirectory()) { + walk(localPath, relPath, linked) + } else if (stat.isFile()) { + tasks.push({ localPath, s3Key: `${s3BasePath}/${relPath}` }) } } + ancestors.delete(dir) } - walk(directoryPath, s3BasePath) + walk(root, "", false) console.log(`Discovered ${tasks.length} files to upload`) @@ -114,9 +168,40 @@ async function uploadDirectoryToS3( requestBody: new Blob([markerBody], { type: "application/json" }), }) console.log(`Wrote completion marker: ${markerKey}`) + + return tasks.length } ``` +## Symlinks + +A link is resolved with `realpathSync` and followed only when its target lies +inside the checkout's real path. A file target is uploaded under the link's own +path; a directory target is walked as if it sat there, so +`inventories/prod/group_vars -> ../../shared/group_vars` shows up in the viewer +with its files. Everything else is skipped and logged: + +- **A target outside the checkout.** The repository chooses the target, and the + checkout sits in the job's working directory next to the ssh key + `get_git_ssh_cmd` writes (`../ssh_id_priv_0`) and the job's `args.json`. A + link to one of those, or to `/proc/self/environ` with the caller's + `WM_TOKEN`, would put it in storage for every reader of the resource. This + is why the walk does not follow links the way `aws s3 sync` does. +- **A target that cannot be resolved**: a dangling link, or a link loop + (`ELOOP`). +- **A directory that is already being walked higher up** (`loop -> .`, + `up -> ..`). The guard holds the real paths of the current descent only, as + `find -L` does, not every directory seen so far: a directory reachable + through two links is uploaded under both paths, as the checkout presents it. +- **Anything reached through a link once the budget is spent.** Because a + directory can be reached along many paths, two links to the next directory + at each level double the tree, and a repository a few dozen links deep would + expand past what the job can hold in memory. Every entry reached through a + link counts against a budget of 20,000 entries and 512 MiB. It is charged + before the link is resolved, so links that end up skipped count too, and + neither their work nor their log lines can multiply. Past the budget, the rest + are skipped with one log line. The checkout's own files are always uploaded. + ## Notes for review - **Concurrency level**: 16 is a starting point; tune based on instance @@ -127,6 +212,12 @@ async function uploadDirectoryToS3( paths on retry, so a partial upload + retry naturally heals. Old commit directories from before this patch are unreachable through the UI but still consume storage; an instance admin can prune them manually if desired. +- **A new pin doesn't refresh commits already uploaded**: the viewer keys + storage on the commit hash (`gitrepos/{workspace}/{resource_path}/{commit_hash}/`) + and only checks that the marker exists. So a commit uploaded by an earlier + script version keeps that version's tree (hub/28905's had no symlinks) until + the repository's head moves to a new commit, or an admin deletes that + commit's directory. - **Error propagation**: keep the existing `try/catch` in `main` so an upload failure surfaces in the job result and is shown in the new viewer error banner. diff --git a/docs/git-sync-gitlab-setup.md b/docs/git-sync-gitlab-setup.md new file mode 100644 index 0000000000..6e736dee37 --- /dev/null +++ b/docs/git-sync-gitlab-setup.md @@ -0,0 +1,170 @@ +# Git sync with GitLab + +GitLab has no equivalent of a GitHub App, so there is nothing to install and no +consent screen. What Windmill needs instead is one credential you create in +GitLab and paste once. With it, a GitLab repository gets the same managed +features an app-backed GitHub repository has: instant pull over a webhook, merge +requests opened on deploy, and a diff preview posted onto the merge request. + +## The credential + +Create a **project access token** on the project you are syncing (Settings → +Access tokens). It is a bot identity that outlives the person who created it, +which is what you want for a credential the instance uses unattended, and it +reaches exactly the one project. + +| | | +| --- | --- | +| Scope | `api` | +| Role | Developer to push deploy branches; **Maintainer** to also manage the webhook and open merge requests | +| Expiry | Required. A group service account PAT can be non-expiring on self-managed (see below); an access token cannot | + +**Use a separate token per repository.** A group access token works too and +reaches every project in the group, which is convenient for a lot of +repositories — but Windmill stores the credential per repository, and renewal +rewrites the repository it renewed for. Any other repository holding that same +token keeps the revoked one and stops syncing until you paste a new token there. +Each stranded repository says so on its card, so it is visible rather than +silent, but a token per repository avoids it entirely. + +`api` is a superset: it authorizes Git over HTTPS as well, so no separate +`write_repository` is needed to clone and push, and it is also what makes the +token renewable. A `write_repository`-only token can still push, but Windmill +cannot inspect or renew it and reports that in the workspace's git sync settings. + +### The identity Windmill acts as + +GitLab issues an access token to a bot user it creates for it — `project__bot_…` +for a project token, `group__bot_…` for a group one — and the bot's display +name is **the name you gave the token**. That name is the byline on everything +Windmill does: the author of deploy commits, of the merge requests it opens, and +of the preview notes it writes. Name it for what it is, `windmill-sync` or +similar, rather than something only you will recognise. + +Each token you create adds another bot member to the project or group. Renewal +does not — it keeps the same bot — so a repository accumulates one bot, not one +per year. + +Renewal goes through GitLab's own self-rotation endpoint. Both kinds of access +token are held as their bot user's personal access token, so the token rotates +itself and Windmill never needs a credential with rights over the project or +group. + +## Connecting a repository + +In the resource form for a `git_repository` resource, use the **GitLab** button: +paste the instance URL and the token, pick a project from the list, and Windmill +keeps the token for you. The resource itself gets the plain remote URL +(`"url": "https://gitlab.com/group/project.git"`), with no credential in it. + +The token is stored encrypted on the workspace, keyed by the repository it was +issued for rather than by the resource naming it, the same way a GitHub App +installation is held against the account it covers. Nothing reads it back out +over the API: the server attaches it when it talks to GitLab, and a sync job +receives it only against its own job token. Repointing a resource's `url` asks +for a different repository's token and finds none, so the edit carries nothing +with it; a repository that genuinely moved needs its token entered again. + +Because the repository is the key, the token is stored the moment you pick the +project, before the resource is saved. Renaming the resource later keeps it, and +cancelling the edit leaves a stored token that nothing uses until some resource +points at that repository again. + +Forks of the workspace read this one copy rather than getting their own, so +renewal reaches all of them at once and the token is not duplicated into every +descendant workspace. + +Treat workspace admin as equivalent to holding the token. An admin of the +workspace, or of any fork below it, can point a repository at a sync script they +wrote and have that job request the credential, exactly as they can for a GitHub +App installation token. Storing it this way keeps it out of the variables API and +out of every fork's own storage; it is not a boundary against the admins of those +workspaces. + +A URL with the token written into it, in the resource or in a secret variable +the resource points at (`"url": "$var:..."`), is a plain git remote: it syncs on +deploy and by polling, and nothing else. Windmill does not know the token is +there, so it registers no webhook, opens no merge request, and neither reports +nor renews its expiry. Use the **GitLab** button to hand the token to Windmill +if you want any of that. + +## Expiry and renewal + +Windmill reads `expires_at` from the token it holds and shows it on the +repository in the workspace's git sync settings. Within three weeks of expiry it +rotates the token through GitLab's own +`POST /personal_access_tokens/self/rotate`, stores the replacement, and verifies +it. Only the token can rotate itself, so one without `api` (or `self_rotate`) is +a permanent warning rather than something Windmill can fix. + +Only the workspace that holds a credential renews it, so one renewal serves the +whole fork chain instead of each fork racing to renew the same token. A fork +reads the parent's without holding one, so it never renews; the parent does, and +every fork sees the replacement at once. + +Rotation is deliberately never retried. GitLab revokes the old token the instant +it issues the replacement, and presenting an already-rotated token to `/rotate` +again is treated as reuse: it revokes **the whole token family, including the +live replacement**. So a rotation that succeeded at GitLab but failed to persist +is surfaced as an error to act on, not retried. + +Non-expiring tokens are possible only for a **group service account PAT** on +self-managed, with `require_personal_access_token_expiry` turned off in the +instance's application settings. A group access token is always rejected without +an `expires_at`. + +## What each managed feature needs + +| Feature | Needs | +| --- | --- | +| Instant pull | A project hook Windmill creates, so Maintainer; and a Windmill base URL GitLab can reach | +| Merge requests on deploy | Developer, plus the `api` scope | +| Diff preview on a merge request | The project hook, plus permission to post merge request notes | + +Instant pull falls back to checking the tracked branch about every minute when +the hook cannot be created or delivered, so nothing silently stops syncing. + +## Self-managed differences + +**Webhooks to a private network are blocked by default.** GitLab refuses to +create a hook pointing at a private or local address until an administrator +enables *Allow requests to the local network from webhooks and integrations* +(Admin → Settings → Network → Outbound requests, +`allow_local_requests_from_web_hooks_and_services`). A Windmill instance on the +same private network as GitLab needs this; without it, hook creation fails with a +"blocked" error and the repository keeps polling. + +**A relative-URL install is not supported.** GitLab can be served under a path +prefix (`https://example.com/gitlab`), and that prefix cannot be told apart from +a group of the same name: `example.com/a/b/c.git` is either group `a/b` project +`c`, or prefix `a` with group `b` project `c`. Windmill reads it as the nested +group, so on a relative-URL install it derives the wrong API base and the managed +features stay unavailable. Such a repository still syncs through its token URL, +which needs no API base. + +Everything else is identical: Windmill talks to `/api/v4` and needs +no inbound access of its own beyond the hook deliveries. + +## The deploy preview is a note, not a pipeline status + +On GitHub the preview is a check run: its own object, advisory unless the +repository makes it required. GitLab has no equivalent. Its only comparable +primitive is a commit status, and posting one has side effects Windmill will not +impose on a project: + +- GitLab files the status **as a job inside whatever pipeline already covers that + commit**, so a failed Windmill status fails the project's own pipeline, and its + reviewers see their test suite as failed. +- `allow_failure` is ignored on the commit-status endpoint, so the status cannot + be made advisory. +- On a commit with no pipeline it creates an `external` pipeline instead, which + then gates merging under *Pipelines must succeed*, including while it is still + running. + +So on GitLab the preview lives entirely in a **merge request note** that Windmill +keeps up to date: it carries the workspace, the status line, the commit, a link +to the job, and the full list of changes merging would deploy. A note cannot +block a merge or change what the project's own CI reports. + +The note is upserted rather than appended, so a merge request accumulates one +Windmill comment however many times it is pushed to. diff --git a/docs/git-sync-pull-design.md b/docs/git-sync-pull-design.md index 8961b0f047..5f77a3f117 100644 --- a/docs/git-sync-pull-design.md +++ b/docs/git-sync-pull-design.md @@ -618,6 +618,93 @@ repo's **Environments** timeline ("Production → Deployed"). Needs opt-in / later. The check-run version is the cheap default and matches the visual Cloudflare parity without a new permission. +### Phase 7 — CI test results check (WIN-2051) — implemented + +Surfaces Windmill's own CI tests (the `// test: script/...` annotation) as a +**"Windmill CI tests"** check run on **any PR** against the tracked branch, so a customer +can mark it a **required status check** and have Windmill CI results gate the PR — +replacing the documented GitHub Action that polls `ci_test_results_batch`. GitHub App-backed +only; reuses the Phase 4 `Checks: write` grant, so no new permission. Token repos keep the +Action, and GitLab merge requests get no CI-test surface for the same reason the Phase 4 +preview lives in a note there (a commit status would fail the project's own pipeline). + +Driven by the **`pull_request` webhook** — the same event Phase 4 already reacts to — +rather than the deploy push/pull, so it's uniform across how the PR's commit came to exist +(a fork deploy that pushes `wm-fork/**` and opens the PR, or an external push that gets +pulled in). CI tests run as separate async `ci_test` jobs in the **fork workspace** the PR +corresponds to; the check reflects that fork's current results on the PR head. + +- **State** — `git_sync_ci_test_check(workspace_id, repo_resource_path, head_sha)` (new + table). `workspace_id` is the **fork** whose `ci_test` jobs the check reflects; + `repo_resource_path` the repository (a fork can sync several, and two can hold the same + commit); `poster_workspace_id` is the **parent** whose GitHub-App installation posts the + run (the workspace that received the webhook and owns the repo hook). Plus `repo_url`, + `head_ref`, `check_run_id` (NULL until the create succeeds, and reset to NULL by a re-fired event + for the same head: the row is written first so a create that never gets recorded cannot + strand an in-progress run, and the poller retries any row without an id), `created_at`, + `concluded`, `conclusion`, `concluded_at`, `github_posted`. + Partial index `(workspace_id) WHERE NOT concluded OR NOT github_posted` (the live set the + hook + poller scan). +- **Open** — in the `pull_request` handler (opened/synchronize/reopened, or edited with a + base change, base = tracked): when the head lives in the base repo, resolve the fork + workspace from the head ref (reusing the fork-branch routing; + `resolve_pr_head_workspace`), persist the intent row with a null check-run id, then + `create_check_run` in_progress on `head_sha` via the parent's installation and adopt the + id (the poller retries the create from the row if it failed), then evaluate. An earlier head's open check is left to conclude on its + own (fork verdict or timeout): a late-delivered event for an old head must never touch + the current head's check. +- **Conclude** — the verdict is the head's own suite. Once the fork reflects the head (below) + and its dependency jobs settled, every CI test the fork declares is dispatched once, one + run per `ci_test_reference` row the way a deploy of that item would (`trigger_all_ci_tests`, + as the fork's owner, the user who created it, with no more reach than they have; a + missing or disabled owner concludes the check as failure; and without the per-item + debounce so a deploy-triggered run of the same test cannot supersede a suite run), and the job ids are + recorded on the synced-head row (`ci_test_job_ids`; `tests_dispatched_at` claims the + dispatch so the per-job hook and the poller queue it once, and a claim that never recorded + ids is retaken after 5 min). The verdict is exactly those runs: fail-fast on any + failed/canceled; `success` once all settle ("No CI tests" when the fork declares none); + `skipped` ignored. Nothing older, newer or workspace-wide stands in + for a head's runs, so a re-fired event reads the same runs and gets the same answer, a + test-only change is run because the suite runs on every head, and a deploy in flight in + the fork cannot feed another head's check. Runs purged by job retention reset the row so + the head is re-tested. +- **Readiness** — the suite is dispatched only once the fork reflects the head, so it does + not matter which webhook GitHub delivers first. The evidence is `git_sync_synced_head`: the + pull completion hook writes a row when a pull job succeeds (the pull script reports the + commit its clone checked out; the enqueue-time marker is the fallback), and the push + completion hook writes one from the deploy push script's `{pushed, sha, branch, rebased}` + result (a rebased push sits on unpulled commits and is not recorded). The head is ready + when the repository branch's newest row names it, so a branch reset to an older commit + waits for its re-pull; the prune keeps each repository branch's newest row so a PR reopened + at an unchanged head stays ready. This is a sync event log, deliberately apart from + `auto_pull.last_synced_sha`: that map decides whether the next poll pulls (a push must + never write it, or a commit someone else pushed under ours would be skipped) and it is + client-round-tripped settings. The check row stores `head_ref` for the lookup. Dispatch + also waits while a dependency job in the fork or a pull of the repository branch is queued + (a deploy push lands on whatever the remote held when it cloned, so a commit pushed there + from outside is in the workspace only once its pull ran), and the check fails outright if + a dependency job failed after the head's pull started (the item deployed nothing + runnable). A commit the fork never comes to reflect times out; a timeout on a repository + pinned to a sync script older than the one that reports pushed commits names that as the + reason. Needs the hub script versions that report the sha (`LATEST_GIT_SYNC_SCRIPT_PATH`, + `GIT_SYNC_PULL_SCRIPT_PATH`). +- **Drivers** — a per-`ci_test`-job completion hook (low latency) and the git-sync poller + (the backstop: retries the GitHub create/deliver, times stuck checks out after 30 min, + prunes old rows; runs after the auto-pull advisory lock is released so its GitHub calls + never extend the tick). Both call one idempotent `evaluate_and_conclude`, which claims the + decision with a guarded `UPDATE ... WHERE NOT concluded RETURNING` (exactly-once) and + decouples GitHub delivery via `github_posted` so a failed PATCH is retried, not hung. + +Invariants: only a head in the base repo can map to a workspace (a contributor fork's +branch names mean nothing here); the webhook's workspace posts through its own +installation; the timeout stops a hung test job from blocking a required check forever; +rows cascade away with either workspace. A plain feature-branch or +contributor-fork PR resolves to no fork workspace and gets an already-concluded `skipped` +check (branch protection counts `skipped` as passing, so requiring the check does not block +those PRs). A timeout on a repository pinned to a sync script older than the one that reports +pushed commits names that as the reason. Known limit (accepted for v1): the fork's status is workspace-wide (all its +tested items), which for the one-fork-per-PR model equals the PR's scope. + ## 16. Alternatives considered **Portal as webhook proxy (the rejected "option 2").** Subscribe the managed app diff --git a/docs/reusable-ai-agents.md b/docs/reusable-ai-agents.md index 8383bc2836..ede8f64b1d 100644 --- a/docs/reusable-ai-agents.md +++ b/docs/reusable-ai-agents.md @@ -22,22 +22,54 @@ every workspace via the standard cached-resource-type sync, like other built-in or flow expressions), so saving round-trips losslessly. Each host flow overrides what it needs: `tool_inputs` stores per-tool overrides (a diff from the resource tool's own transforms) that overlay onto the matching tools at runtime. Editing on a linked step edits - the flow's use of the agent; editing under the "Editing" banner edits the agent itself. + the flow's use of the agent; editing in the agent editor edits the agent itself. In the flow editor, the AI agent step's **Step Input** tab shows a single read-only card (*linked to *, with the inherited brain + tools and an explanatory tooltip) plus -*Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink* -(fork the resolved config — including any `tool_inputs` — back into the step as a one-off). -While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking -first when there is something to drop), and the unsaved-changes badge opens a diff against the -deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of, -and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so -a reload brings the step back as a standalone agent with no path to save back to. +*Edit*, which opens the agent editor over the flow, and *Unlink* (fork the resolved config — +including any `tool_inputs` — back into the step as a one-off). A linked agent's tools appear as display-only graph tool nodes (clicking one selects the agent step); below the step's inputs, each tool gets a section with the standard schema-aware input editors (prop picker included) and a read-only view of its code — edits persist into `tool_inputs`. +## Drafts + +The agent editor edits the resource through a **per-user resource draft** (`draft` table, +`item_kind = 'resource'`), autosaved by `useAgentDraft` and deployed by the editor's own Deploy +button. It is the same draft row the generic resource editor writes and the Review & Deploy page +lists, so an agent can be deployed from any of them. + +A flow does not wait for that deploy to see the draft: + +- Testing the flow, or a single linked step, runs the draft. `runFlowPreview` and `ModuleTest` + substitute each linked step for the standalone step the draft would run as + (`linkedAgentDrafts.ts`): `agent` cleared, the draft's brain as static input transforms, the + draft's tools on the step, and the step's own `user_message`/`user_attachments` kept on top — + the same overlay order `ai_executor.rs` applies to a linked step. `tool_inputs` is untouched, + since the worker overlays it in both branches. +- The step's linked card and the graph's tool nodes show the draft, with a *Draft* badge, so the + editor describes what a test would run. Read-only surfaces (the deployed flow page, the run + viewer) stay on the deployed agent: they resolve tools through `publishLinkedAgentTools` without + the draft flag. +- Deploying the flow lists every linked agent that has a draft in the confirmation dialog, beside + the draft triggers. Deploying one writes the resource and drops the draft; leaving one out keeps + its draft untouched, and the flow runs the agent as currently deployed. That is the one place + the two kinds differ: an undeployed draft trigger is deleted, because it belongs to the flow, + while an agent draft belongs to a resource other flows also use. + +Because a draft is per-user, a flow test can behave differently for two people looking at the same +flow. That is the same contract as a flow draft, and deploying the agent is what makes it shared. + +Inlining has a consequence worth knowing: a preview job's `raw_flow` then carries the agent's +config, where a linked step used to carry only the path and leave the resolution to the worker. So +an agent's prompt and tool set are readable by whoever can read that preview job, which is a wider +set than whoever can read the resource when the agent sits in a more restricted folder than the +flow. No credential travels with it — the provider stays a `$res:` reference, resolved at run time +as the runner. The agent editor's own test pane has inlined the same way since drafts existed; +closing the gap would mean the preview carrying a draft *reference* the worker resolves, rather +than the config. + Sharing works through standard resource folder permissions (save agents under `f/...`). Only the agent's brain is interpolated when the step runs. A tool's own `$res:`/`$var:` defaults are diff --git a/docs/wac-sdk-e2e.md b/docs/wac-sdk-e2e.md index 5c6682d93d..0920bdf168 100644 --- a/docs/wac-sdk-e2e.md +++ b/docs/wac-sdk-e2e.md @@ -34,8 +34,12 @@ cp python-client/wmill/wmill/client.py \ /tmp/windmill-mytest/cache/python_3_12/wmill==*/wmill/client.py find /tmp/windmill-mytest -name __pycache__ -type d -exec rm -rf {} + -# 4. RESTART the backend — see below -# 5. run your scenarios, and rm -rf /tmp/windmill-mytest when done +# 4. drop the bundle snapshots — a bun job runs a bundle built from the package, +# cached by content hash, so a job that already ran keeps the old SDK inlined +rm -rf /tmp/windmill-mytest/cache/bun + +# 5. RESTART the backend — see below +# 6. run your scenarios, and rm -rf /tmp/windmill-mytest when done ``` ## Restart the workers after injecting diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6fd6413d59..cc7a8ee426 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.803.0", + "version": "1.811.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.803.0", + "version": "1.811.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -5579,9 +5579,9 @@ } }, "node_modules/driver.js": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.3.6.tgz", - "integrity": "sha512-g2nNuu+tWmPpuoyk3ffpT9vKhjPz4NrJzq6mkRDZIwXCrFhrKdDJ9TX5tJOBpvCTBrBYjgRQ17XlcQB15q4gMg==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.8.0.tgz", + "integrity": "sha512-+8/IO7h1v14IzWh2GP60N7T3PFZweXwdn5e5POuxRSBoCYUojsBxzqawPeXh3YZIibRy7EehYNEyxe7slwwtdg==", "license": "MIT" }, "node_modules/dts-bundle-generator": { diff --git a/frontend/package.json b/frontend/package.json index f8f952ede6..819a45ca8f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.803.0", + "version": "1.811.1", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/actingUser.svelte.ts b/frontend/src/lib/actingUser.svelte.ts new file mode 100644 index 0000000000..30da65ff83 --- /dev/null +++ b/frontend/src/lib/actingUser.svelte.ts @@ -0,0 +1,74 @@ +import { untrack } from 'svelte' +import { fromStore } from 'svelte/store' +import { SvelteMap } from 'svelte/reactivity' +import { userStore, workspaceStore, type UserExt } from '$lib/stores' +import { getWorkspaceRole, type RoleLookup } from '$lib/user' + +/** + * The user acting in a workspace that is not necessarily the one the top nav points at — an AI + * session or a workspace-specific variant acts on a workspace the nav deliberately is not on. + * + * `$userStore` answers for the navigation workspace at no cost, exactly as every permission check + * in the app did before this hook existed — including when it holds nobody, which reads as unknown + * and refuses. Every other workspace is looked up, and an unresolved user there is `undefined`: it + * must never fall back to the navigation user, whose rights belong to another workspace. + * `canWrite`/`isOwner` refuse for an unknown user, which is the only safe answer. A caller that + * must not render that refusal as a denial asks `resolved` first. + */ +export function useActingUser(workspace: () => string | undefined) { + const navWorkspace = fromStore(workspaceStore) + const navUser = fromStore(userStore) + const looked = new SvelteMap() + // The workspace this effect last acted on, so arriving at one is distinguishable from the + // effect re-running while already there. + let asking: string | undefined + + $effect(() => { + const ws = workspace() + if (asking !== ws) { + asking = ws + // Dropped on the way *in*, not on the way out: a lookup that fails after the acting + // workspace has already moved on has no entry to clear at the moment it is left, so + // clearing it there would keep a refusal that no attempt is behind any more. + if (ws && untrack(() => looked.get(ws)?.kind) === 'lookup_failed') looked.delete(ws) + } + if (!ws || ws === navWorkspace.current) return + // Any settled answer stops the asking, a failure included — otherwise recording one + // would re-enter this effect and loop. + if (looked.has(ws)) return + untrack(() => { + // Memoized process-wide, so two components pointed at the same workspace share one + // request rather than each issuing their own. + getWorkspaceRole(ws).then((lookup) => looked.set(ws, lookup)) + }) + }) + + function userIn(ws: string | undefined): UserExt | undefined { + if (!ws) return undefined + if (ws === navWorkspace.current) return navUser.current + const lookup = looked.get(ws) + return lookup?.kind === 'resolved' ? lookup.user : undefined + } + + return { + /** The acting user in `ws`, or `undefined` when it is not known. Only workspaces this + * hook has been pointed at are looked up; the rest read as unknown. */ + in: userIn, + /** Whether `ws` has an answer at all — a user, or a lookup that came back without one. + * The navigation workspace always has one: `$userStore`, "nobody" included. */ + resolved: (ws: string | undefined): boolean => + !!ws && (ws === navWorkspace.current || looked.has(ws)), + get current(): UserExt | undefined { + return userIn(workspace()) + }, + /** Drop the lookups that came back empty so they are asked again. Arriving at a + * workspace already does this; a long-lived editor must call this too when it starts a + * fresh session on the workspace it is already on, or a `whoami` that happened to fail + * pins it to "unknown user" for as long as it stays there. */ + forgetFailures(): void { + for (const [ws, lookup] of looked) { + if (lookup.kind === 'lookup_failed') looked.delete(ws) + } + } + } +} diff --git a/frontend/src/lib/assets/app.css b/frontend/src/lib/assets/app.css index 760f4c0261..f6dc648234 100644 --- a/frontend/src/lib/assets/app.css +++ b/frontend/src/lib/assets/app.css @@ -209,33 +209,24 @@ U+1fac6, U+1fae0-1fae6, U+1fae8-1faea, U+1faef-1faf8; } - .prose-xs ul { - margin-top: 0.5rem; - list-style-type: '- '; - padding-left: 1.5rem; - } - + /* Bullets read as a dash rather than a disc. Only the glyph is overridden: + indentation and vertical rhythm stay with Tailwind Typography so ordered + and unordered lists line up with each other. */ .prose ul { - margin-top: 1.5rem; list-style-type: '- '; - padding-left: 3rem; } - /* The '- ' list markers, horizontal rules and blockquote bars otherwise - fall through to Tailwind Typography's default bullet/border colors, which - are nearly invisible on dark backgrounds (e.g. the AI chat). Use - theme-aware tokens so they stay readable in both light and dark mode. */ - .prose-xs ul > li::marker, - .prose ul > li::marker { + /* List markers, horizontal rules and blockquote bars take the tertiary/light + tokens the typography config maps them to, which is too faint to read on + the denser markdown surfaces (e.g. the AI chat). Step them up one. */ + .prose :is(ul, ol) > li::marker { color: rgb(var(--color-text-secondary)); } - .prose-xs hr, .prose hr { border-top-color: rgb(var(--color-border-normal)); } - .prose-xs blockquote, .prose blockquote { border-left-color: rgb(var(--color-border-normal)); } diff --git a/frontend/src/lib/attachments/newTabModifier.dom.test.ts b/frontend/src/lib/attachments/newTabModifier.dom.test.ts new file mode 100644 index 0000000000..115877298c --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.dom.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { newTabModifier } from './newTabModifier.svelte' + +const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent }) +const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)' +const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + +const attached: (() => void)[] = [] + +/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */ +function pill() { + const node = document.createElement('span') + document.body.append(node) + const modifier = newTabModifier() + const cleanup = modifier.attach(node) as () => void + attached.push(cleanup) + const hover = (init: MouseEventInit = {}) => + node.dispatchEvent(new MouseEvent('mouseenter', init)) + const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init)) + const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave')) + return { modifier, hover, move, unhover, cleanup } +} + +const keydown = (init: KeyboardEventInit) => + window.dispatchEvent(new KeyboardEvent('keydown', init)) + +describe('newTabModifier', () => { + // The window listeners outlive the DOM, so every case has to be torn down through the + // attachment rather than by emptying the body. + afterEach(() => { + attached.splice(0).forEach((cleanup) => cleanup()) + document.body.replaceChildren() + vi.unstubAllGlobals() + }) + + // The hover event carries the live modifier state, so a modifier pressed before the pointer + // arrived (or while this window was unfocused) is picked up rather than read as false. + it('seeds from the hover event, per platform', () => { + onPlatform(LINUX) + const linux = pill() + linux.hover({ ctrlKey: true }) + expect(linux.modifier.held).toBe(true) + + onPlatform(MAC) + const mac = pill() + // macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier. + mac.hover({ ctrlKey: true }) + expect(mac.modifier.held).toBe(false) + mac.hover({ metaKey: true }) + expect(mac.modifier.held).toBe(true) + }) + + // Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase + // listener would go blind whenever focus sits in one. + it('sees a keydown that a focused element stops from propagating', () => { + onPlatform(LINUX) + const { modifier, hover } = pill() + hover() + const input = document.createElement('input') + input.addEventListener('keydown', (e) => e.stopPropagation()) + document.body.append(input) + + input.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true }) + ) + expect(modifier.held).toBe(true) + }) + + // A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown + // on the way back, while the pointer parked on the pill fires no fresh mouseenter either. + it('re-seeds from pointer movement after the window lost focus', () => { + onPlatform(LINUX) + const { modifier, hover, move } = pill() + hover({ ctrlKey: true }) + window.dispatchEvent(new Event('blur')) + expect(modifier.held).toBe(false) + + move({ ctrlKey: true }) + expect(modifier.held).toBe(true) + }) + + it('stops tracking once unhovered', () => { + onPlatform(LINUX) + const { modifier, hover, unhover } = pill() + hover({ ctrlKey: true }) + unhover() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) + + it('stops tracking when the element is destroyed while hovered', () => { + onPlatform(LINUX) + const { modifier, hover, cleanup } = pill() + hover({ ctrlKey: true }) + // Hovering again without leaving must not strand the first hover's listeners, which nothing + // would then hold a reference to. + hover({ ctrlKey: true }) + cleanup() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) +}) diff --git a/frontend/src/lib/attachments/newTabModifier.svelte.ts b/frontend/src/lib/attachments/newTabModifier.svelte.ts new file mode 100644 index 0000000000..06c69f36cc --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.svelte.ts @@ -0,0 +1,66 @@ +import type { Attachment } from 'svelte/attachments' +import { isMac } from '$lib/utils' + +/** + * Tracks whether the modifier that turns a click into a new browser tab is held, but only while + * the attached element is hovered, which is the only moment the answer is used. + */ +export function newTabModifier() { + let held = $state(false) + + // Only the modifier that actually yields a tab: shift opens a window, alt can start a + // download, and on macOS ctrl+click is a secondary click. + // Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to + // a focus change cannot strand the flag on. + const sync = (event: KeyboardEvent | MouseEvent) => { + held = isMac() ? event.metaKey : event.ctrlKey + } + const clear = () => { + held = false + } + + const attach: Attachment = (node) => { + // One controller per hover: a mirrored remove list leaks any listener whose options drift. + let hover: AbortController | undefined + const leave = () => { + hover?.abort() + hover = undefined + clear() + } + const enter = (event: MouseEvent) => { + // Seeded from the hover itself: mouse events carry the same modifier flags as key events, + // so a modifier already held before the pointer arrived reads correctly. + sync(event) + // Re-entering without an intervening leave would strand the previous controller: nothing + // else references it, so its listeners could never be removed. + hover?.abort() + hover = new AbortController() + const { signal } = hover + // Same reason the hover seeds: a modifier held across a keyboard app switch delivers no + // keydown on the way back, so the pointer is all that is left to re-read it from. + node.addEventListener('mousemove', sync, { signal }) + // Capture: editors and menus stopPropagation the keys they handle, hiding the modifier + // from a bubble-phase listener whenever focus sits in one. + window.addEventListener('keydown', sync, { capture: true, signal }) + window.addEventListener('keyup', sync, { capture: true, signal }) + // Not capture, unlike the two above: blur does not bubble but does reach the window while + // capturing, so it would fire for every element that loses focus. + window.addEventListener('blur', clear, { signal }) + } + + const life = new AbortController() + node.addEventListener('mouseenter', enter, { signal: life.signal }) + node.addEventListener('mouseleave', leave, { signal: life.signal }) + return () => { + life.abort() + leave() + } + } + + return { + get held() { + return held + }, + attach + } +} diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index 259d9c1cf9..0d76b1a0b8 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -35,6 +35,8 @@ export interface SchemaProperty { } min?: number max?: number + /** Height a string field's text area opens at, in rows. */ + minRows?: number currency?: string currencyLocale?: string multiselect?: boolean diff --git a/frontend/src/lib/components/AIProviderPicker.svelte b/frontend/src/lib/components/AIProviderPicker.svelte index 4392a2d513..19b1346b84 100644 --- a/frontend/src/lib/components/AIProviderPicker.svelte +++ b/frontend/src/lib/components/AIProviderPicker.svelte @@ -4,11 +4,7 @@ import { fetchAvailableModels, AI_PROVIDERS } from './copilot/lib' import type { AIProvider, ProviderConfig } from '$lib/gen' import { workspaceStore } from '$lib/stores' - import { get } from 'svelte/store' - import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' - import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import ResourcePicker from './ResourcePicker.svelte' - import ToggleButtonMore from './common/toggleButton-v2/ToggleButtonMore.svelte' import Toggle from './Toggle.svelte' import { saveConfig, removeConfig, isSameAsStoredConfig } from './aiProviderStorage' import AIReasoningEffortPicker from './AIReasoningEffortPicker.svelte' @@ -17,9 +13,20 @@ value: ProviderConfig | undefined disabled?: boolean actions?: Snippet + /** The workspace the surface operates on, which a session or fork editor sets to something + * other than the one being navigated. Resources and the models read off them are per + * workspace, so without it this offers what the wrong one holds. */ + workspace?: string | undefined } - let { value: _uncheckedValue = $bindable(), disabled = false, actions }: Props = $props() + let { + value: _uncheckedValue = $bindable(), + disabled = false, + actions, + workspace = undefined + }: Props = $props() + + let effectiveWorkspace = $derived(workspace ?? $workspaceStore ?? '') let value = $derived.by(() => { if (!_uncheckedValue || typeof _uncheckedValue !== 'object') return undefined @@ -30,7 +37,13 @@ let availableModels = $state([]) let filterText = $state('') - let modelsCache = new Map() + // Keyed by provider *and* path: two `customai` resources point at different base URLs, so they + // do not share a model list. + let modelsCache = new Map() + + // The resource picker offers every provider type at once and the pick is what names the kind. + // One string for the component's life: it is what the picker queries with. + const providerResourceTypes = Object.keys(AI_PROVIDERS).join(',') if (!_uncheckedValue) { _uncheckedValue = { @@ -57,12 +70,6 @@ return r }) - // Provider options for the toggle button group - const providerOptions = Object.entries(AI_PROVIDERS).map(([key, details]) => ({ - value: key as AIProvider, - label: details.label - })) - async function loadModels(signal?: AbortSignal) { const provider = value?.kind const resourceValue = value?.resource @@ -73,20 +80,20 @@ } loading = true - if (modelsCache.has(provider)) { - availableModels = modelsCache.get(provider) || [] + const cacheKey = `${effectiveWorkspace}:${provider}:${resourcePath}` + if (modelsCache.has(cacheKey)) { + availableModels = modelsCache.get(cacheKey) || [] loading = false return } try { - const workspace = get(workspaceStore) || '' - const models = await fetchAvailableModels(resourcePath, workspace, provider, signal) + const models = await fetchAvailableModels(resourcePath, effectiveWorkspace, provider, signal) if (signal?.aborted) { return } availableModels = models - modelsCache.set(provider, models) + modelsCache.set(cacheKey, models) } catch (e) { if (signal?.aborted) { return @@ -101,15 +108,24 @@ } } - // Handle provider selection - function onProviderChange(selectedProvider: AIProvider) { - if (value) { - value.kind = selectedProvider - value.resource = '' - value.model = '' - // Reasoning effort is model-specific; reset it with the model. - value.reasoning_effort = undefined + /** + * The provider kind follows the resource that was picked. Driven by the pick rather than by an + * effect on the picker's `valueType`, which also resolves for the value the field was opened on + * and would rewrite a saved config just for being looked at. + */ + function onResourcePicked(_path: string | undefined, type: string | undefined) { + // An empty type is the placeholder the picker keeps for a saved path it could not find. It + // says nothing about the provider, so the kind stands. + if (!value || !type || !(type in AI_PROVIDERS)) { + return } + if (value.kind === type) { + return + } + value.kind = type as AIProvider + // Models are per provider, and a reasoning token is per model. + value.model = '' + value.reasoning_effort = undefined } // Helper functions to handle $res: prefix like ObjectResourceInput does @@ -165,97 +181,74 @@ }) -
- - - {#snippet children({ item })} - {#each providerOptions.slice(0, 3) as option} - - {/each} - p.value === value?.kind) >= 3 ? '' : 'More'} - togglableItems={providerOptions.slice(3)} - {item} - bind:selected={() => value?.kind, (v) => v && onProviderChange(v)} - /> - {/snippet} - - - -
-
-

resource

- resourceValueToPath(value?.resource), - (v) => { - if (value) { - value.resource = pathToResourceValue(v) ?? '' - } +
+
+ Resource + + resourceValueToPath(value?.resource), + (v) => { + if (value) { + value.resource = pathToResourceValue(v) ?? '' } } - resourceType={value?.kind} - disabled={disabled || !value?.kind} - placeholder="Select resource" - selectFirst={true} - /> -
+ } + resourceType={providerResourceTypes} + {disabled} + {workspace} + placeholder="Select an AI provider resource" + selectFirst={false} + onValueChange={onResourcePicked} + /> +
- +
+ Model + value?.model, (v) => value && (value.model = v ?? '')} - placeholder="Select model" - disabled={disabled || !value?.kind || !resourceValueToPath(value?.resource)} - onCreateItem={(r) => { - availableModels.push(r) - if (value) value.model = r - }} - createText="Press enter to use custom model" - {loading} - clearable={false} - noItemsMsg={'No models available'} - bind:filterText + Reasoning effort + value?.reasoning_effort, (v) => value && (value.reasoning_effort = v)} + providerConfig={value} + {disabled} />
+ {/if} - - {#if value?.model} -
-

reasoning effort

- value?.reasoning_effort, (v) => value && (value.reasoning_effort = v)} - providerConfig={value} - {disabled} - /> -
- {/if} - - -
- { - if (!e.detail) { - removeConfig() - } else { - saveConfig(value) - } - }} - /> -
+
+ { + if (!e.detail) { + removeConfig() + } else { + saveConfig(value) + } + }} + />
{@render actions?.()} diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index e4ee3ca7b9..dedeee90e2 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -12,6 +12,7 @@ import { Loader2 } from 'lucide-svelte' import { untrack } from 'svelte' import GitHubAppIntegration from './GitHubAppIntegration.svelte' + import GitLabIntegration from './GitLabIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' import ResourceGen from './copilot/ResourceGen.svelte' @@ -28,6 +29,13 @@ isValid?: boolean linkedSecretCandidates?: string[] | undefined description?: string | undefined + /** Workspace the resource is being saved into, which is not always the one + * being navigated. The GitLab picker has to store the credential where the + * resource will look for it. */ + workspace?: string + /** Fired once the GitLab picker has stored the picked project's token, so a + * form that would otherwise file the URL as a secret knows it holds none. */ + onCredentialStored?: () => void onSynced?: () => void } @@ -39,6 +47,8 @@ isValid = $bindable(true), linkedSecretCandidates = undefined, description = $bindable(undefined), + workspace = undefined, + onCredentialStored, onSynced = undefined }: Props = $props() @@ -249,6 +259,19 @@ }} onDescriptionUpdate={(newDescription) => (description = newDescription)} /> + + { + args = newArgs + rawCode = JSON.stringify(args, null, 2) + rawCodeEditor?.setCode(rawCode) + }} + />
{#if resourceType?.includes('bedrock') && !isCloudHosted()} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index a9a0837d1f..99adeac6b3 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -21,9 +21,10 @@ } from '$lib/gen' import { emptyString, truncateRev, urlize } from '$lib/utils' import { registryEntryFor, registryCcCapableFor, stripSandboxSuffix } from './oauthRegistry' - import { createEventDispatcher, onDestroy, tick, untrack } from 'svelte' + import { createEventDispatcher, onDestroy, tick } from 'svelte' import Path from './Path.svelte' - import { Button, RadioCard, Skeleton } from './common' + import { ListRow, RadioCard, Skeleton } from './common' + import { useListHighlight } from './common/listRow/listHighlight.svelte' import ApiConnectForm from './ApiConnectForm.svelte' import SearchItems from './SearchItems.svelte' import WhitelistIp from './WhitelistIp.svelte' @@ -40,9 +41,16 @@ import TextInput from './text_input/TextInput.svelte' import { sameTopDomainOrigin } from '$lib/cookies' import SyncResourceTypes from './SyncResourceTypes.svelte' + import { + alphabetical, + byPopularity, + hubResourceTypePicks, + localResourceTypeCounts, + recordHubResourceTypePick + } from './pickerPopularity' import Label from './Label.svelte' import ResourcePathHint from './ResourcePathHint.svelte' - import { twMerge } from 'tailwind-merge' + import SchemaForm from './SchemaForm.svelte' interface Props { step?: number @@ -223,6 +231,28 @@ | undefined ) + /** Fields of the resource type the provider's registry entry asks for once the token is + * in (`resource_fields`): what no token response carries, like Snowflake's database. A + * list rather than "every other field" because most OAuth types also hold the fields of + * another way in: ServiceNow's basic-auth password, Bitbucket's app password. */ + let resourceFields = $derived((registryEntry()?.resource_fields as string[] | undefined) ?? []) + + /** Their slice of the resource type's schema, so they render with the type's own + * descriptions; plain text inputs while the type is not synced from the hub. */ + let resourceFieldsSchema = $derived.by(() => { + const props: Record = + (resourceTypeInfo?.schema as any)?.properties ?? {} + return { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + order: resourceFields, + properties: Object.fromEntries( + resourceFields.map((f) => [f, props[f] ?? { type: 'string', description: '' }]) + ), + required: [] + } + }) + /** Instance entry declares client credentials but not authorization_code * (custom provider configured with only a token URL) */ let authCodeUnavailable = $state(false) @@ -332,6 +362,7 @@ export async function open(rt?: string) { if (!rt) { loadResourceTypes() + loadPopularity() } step = 1 //express && !manual ? 3 : 1 // The list is keyboard-driven from the search field, so it takes focus on open. @@ -378,12 +409,29 @@ } } + /** + * Orders the browse list: the types this workspace already has resources of lead, ranked + * among themselves by the hub's pick counts, then everything else on the same counts. + * `byPopularity` carries the full rule. Both signals are fetched, so the rows render + * alphabetically and re-sort when this lands. + */ + let popularity: (a: string, b: string) => number = $state(alphabetical) + + async function loadPopularity() { + if (!effectiveWorkspace) return + const [hub, local] = await Promise.all([ + hubResourceTypePicks(effectiveWorkspace), + localResourceTypeCounts(effectiveWorkspace) + ]) + popularity = byPopularity(hub, local) + } + async function loadConnects() { if (!connects) { try { - const list = (await OauthService.listOauthConnects()) - .filter((x) => x.name != 'supabase_wizard') - .sort((a, b) => a.name.localeCompare(b.name)) + const list = (await OauthService.listOauthConnects()).filter( + (x) => x.name != 'supabase_wizard' + ) connects = list.map((x) => x.name) connectsInfo = Object.fromEntries(list.map((x) => [x.name, x])) } catch (e) { @@ -466,19 +514,17 @@ // providers — so any of them can also be connected with the user's own // credentials or manually, not only via the shared instance setup (same as // the authorization-code behavior). - connectsManual = availableRts - .map( - (x) => - ({ - key: x, - ...(apiTokenApps[x] ?? { - instructions: '', - img: undefined, - linkedSecret: undefined - }) - }) as { key: string; img?: string; instructions: string[] } - ) - .sort((a, b) => a.key.localeCompare(b.key)) + connectsManual = availableRts.map( + (x) => + ({ + key: x, + ...(apiTokenApps[x] ?? { + instructions: '', + img: undefined, + linkedSecret: undefined + }) + }) as { key: string; img?: string; instructions: string[] } + ) const filteredNativeLanguages = filteredConnectsManual?.filter( (o) => nativeLanguagesCategory?.includes(o[0]) ?? false ) @@ -623,9 +669,11 @@ export async function next() { if (step == 1) { linkedSecrets = [] + // Both branches: the OAuth one fills `resourceFields` into the same map, and fields + // typed into another type's form before Back would otherwise ride along. + args = {} if (manual) { getResourceTypeInfo() - args = {} } else { getResourceTypeInfo() // Awaited: the popup is built from `scopes`, so advancing before this @@ -864,10 +912,19 @@ ) } - const resourceValue = args + // A copy: the form is still mounted and bound to `args` across the awaits below, and + // puts back the default of any field removed from it. + const resourceValue = $state.snapshot(args) let savedVariableCount = 0 if (!manual) { + // A field left blank is absent, not an empty string a consumer reads as a value: + // the Snowflake executor sends any `database` it finds, empty or not. + for (const f of resourceFields) { + if (resourceValue[f] === '' || resourceValue[f] == undefined) { + delete resourceValue[f] + } + } // OAuth flow: single secret variable for the token if (typeof value == 'string' && value != '' && !value.startsWith('$var:')) { savedVariableCount++ @@ -952,6 +1009,10 @@ } }) } + // Saving is what "picking a type" means to the hub: reaching step 2 is still + // browsing. Both branches above count, `filling` included — an imported stub is + // a type taken into the workspace just the same. + recordHubResourceTypePick(effectiveWorkspace, resourceType) dispatch('refresh', path) dispatch('close') sendUserToast( @@ -972,6 +1033,9 @@ if (step == 1) { loadConnects() loadResourceTypes() + // Opened on a specific type, `open()` skipped this; backing out to the browse + // list is the first time it is needed. + loadPopularity() } } @@ -980,24 +1044,35 @@ let filteredConnects: { key: string }[] = $state([]) let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([]) - // uFuzzy scores the name and the description as one string, so searching "google" ranks - // every type whose description mentions Google alongside the ones named after it. Re-sort - // on which field matched, keeping uFuzzy's order within a tier. + let searching = $derived(filter.trim() !== '') + + // Searching, the query owns the order: uFuzzy scores the name and the description as one + // string, so "google" ranks every type whose description mentions Google alongside the + // ones named after it — re-sort on which field matched, keeping uFuzzy's order within a + // tier. Browsing, there is no query to rank against, so popularity orders the list. const rank = (items: { key: string }[] | undefined) => items && - sortResourceTypesByMatch( - items, - filter, - (x) => x.key, - (x) => resourceTypeDescriptions[x.key] - ) + (searching + ? sortResourceTypesByMatch( + items, + filter, + (x) => x.key, + (x) => resourceTypeDescriptions[x.key] + ) + : // Both signals are keyed by resource type, and a sandbox client is a second row + // against one (`salesforce_sandbox` saves a `salesforce`), so it ranks on the + // parent's popularity. Its own key still breaks the tie the pair then have, or + // the two would order arbitrarily. + [...items].sort( + (a, b) => + popularity(stripSandboxSuffix(a.key), stripSandboxSuffix(b.key)) || + a.key.localeCompare(b.key) + )) let rankedConnects = $derived(rank(filteredConnects)) let rankedConnectsManual = $derived( rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined ) - let searching = $derived(filter.trim() !== '') - // Browsing, the "Others" list leads with the native database types. Searching, that // grouping would outrank the search itself — `ms_sql_server` sorting under `mysql` on // "sql" — so the ranked order stands on its own. @@ -1027,15 +1102,8 @@ // Both lists start undefined and render skeletons; "nothing found" only means something // once they have landed. let listsLoaded = $derived(rankedConnectsManual !== undefined && rankedConnects !== undefined) - let highlightedIndex = $state(-1) const rowDomId = (index: number) => `resource-type-row-${index}` - // Set at hover time rather than up front, so only the descriptions the row actually cut - // off carry a tooltip. - function titleIfTruncated(e: MouseEvent & { currentTarget: HTMLElement }) { - const el = e.currentTarget - el.title = el.scrollWidth > el.clientWidth ? (el.textContent?.trim() ?? '') : '' - } const oauthRowOffset = $derived(customKeys.length) const otherRowOffset = $derived(customKeys.length + (rankedConnects?.length ?? 0)) @@ -1054,53 +1122,23 @@ return best } - // Filtering reshuffles the rows under the highlight: point it at the best match so Enter - // takes the top hit, and drop it entirely once the filter is cleared. - $effect(() => { - navItems - filter - untrack(() => (highlightedIndex = searching ? bestMatchIndex() : -1)) + const highlight = useListHighlight({ + count: () => navItems.length, + rowId: rowDomId, + // Sections are rendered in a fixed order, so the best match is not necessarily the + // first row; Enter should still take the top hit. + restingIndex: () => (searching ? bestMatchIndex() : -1), + onActivate: (index) => { + const item = navItems[index] + if (!item) return + item.oauth ? connectOauth(item.key) : selectFromOthers(item.key) + }, + activateEnterFrom: [SEARCH_INPUT_ID] }) - // Scrolling rows under a resting pointer makes the browser fire `mouseenter` on each one, - // which would drag the highlight back under the cursor as the arrow keys move it. Only a - // real pointer move hands the highlight back to the mouse. - let pointerOwnsHighlight = $state(true) - - function highlightHovered(index: number) { - if (pointerOwnsHighlight) highlightedIndex = index - } - - function moveHighlight(delta: number) { - const count = navItems.length - if (count === 0) return - pointerOwnsHighlight = false - // Rows are tabbable buttons, so focus can sit on one. Enter then activates whatever is - // focused, which has to stay the highlighted row. - const rowWasFocused = document.activeElement?.id?.startsWith('resource-type-row-') ?? false - highlightedIndex = - highlightedIndex < 0 - ? delta > 0 - ? 0 - : count - 1 - : (highlightedIndex + delta + count) % count - const row = document.getElementById(rowDomId(highlightedIndex)) - row?.scrollIntoView({ block: 'nearest' }) - if (rowWasFocused) row?.focus() - } - function onListKeydown(e: KeyboardEvent) { if (step !== 1) return - if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { - e.preventDefault() - moveHighlight(e.key === 'ArrowDown' ? 1 : -1) - } else if (e.key === 'Enter' && (e.target as HTMLElement)?.id === SEARCH_INPUT_ID) { - // A focused row activates itself on Enter; this covers Enter typed in the search field. - const item = navItems[highlightedIndex] - if (!item) return - e.preventDefault() - item.oauth ? connectOauth(item.key) : selectFromOthers(item.key) - } + highlight.onKeydown(e) } let editScopes = $state(false) @@ -1132,7 +1170,7 @@
(pointerOwnsHighlight = true)} + onpointermove={highlight.pointerMoved} >
@@ -1146,28 +1184,6 @@
- {#snippet resourceRow(key: string)} -
-
- -
-
-
- {resourceTypeDisplayName(key)} - {key} -
- {#if resourceTypeDescriptions[key]} - - {plainDescription(resourceTypeDescriptions[key])} - - {/if} -
-
- {/snippet} - {#snippet sectionHeading(title: string, count: number)}

{title}{#if searching}{count}{/if} @@ -1175,26 +1191,29 @@ {/snippet} {#snippet resourceButton(key: string, index: number, oauth: boolean)} - + {icon} + {title} + subtitle={resourceTypeDescriptions[key] ? subtitle : undefined} + highlighted={index === highlight.index} + onMouseEnter={() => highlight.hovered(index)} + onClick={() => (oauth ? connectOauth(key) : selectFromOthers(key))} + /> {/snippet}
@@ -1213,7 +1232,7 @@ {#if customKeys.length > 0}
{@render sectionHeading('Custom resource types', customKeys.length)} -
+
{#each customKeys as key, i} {@render resourceButton(key, i, false)} {/each} @@ -1227,7 +1246,7 @@ 'Instance-configured OAuth APIs', rankedConnects?.length ?? 0 )} -
+
{#if rankedConnects} {#each rankedConnects as { key }, i} {@render resourceButton(key, oauthRowOffset + i, true)} @@ -1259,7 +1278,7 @@
{/if} -
+
{#if rankedConnectsManual} {#each otherKeys as key, i} {@render resourceButton(key, otherRowOffset + i, false)} @@ -1388,6 +1407,15 @@ {linkedSecretCandidates} {resourceType} {resourceTypeInfo} + workspace={effectiveWorkspace} + onCredentialStored={() => { + // `forceSecretValue` files a git_repository's `url` in a secret + // variable, for the URLs that carry a token in them. The picker's + // does not: the token is stored separately, so that variable would + // hold nothing secret and add a second place to keep in step with + // the resource. + linkedSecrets = linkedSecrets.filter((f) => f !== 'url') + }} bind:args bind:isValid onSynced={getResourceTypeInfo} @@ -1532,7 +1560,7 @@ > {#if editScopes} - + {:else}
{#each scopes as scope} @@ -1568,6 +1596,17 @@ {/if} + + {#if step == 4 && !manual && !express && !fillPath && resourceFields.length > 0} + + {/if} {#if apiTokenApps[resourceType] || !manual}
  • diff --git a/frontend/src/lib/components/AppTutorials.svelte b/frontend/src/lib/components/AppTutorials.svelte deleted file mode 100644 index 25d06e0167..0000000000 --- a/frontend/src/lib/components/AppTutorials.svelte +++ /dev/null @@ -1,29 +0,0 @@ - - - diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 8550a58b57..b7001059c2 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -1078,6 +1078,7 @@ {otherArgs} {helperScript} {workspace} + {disabled} bind:value format={format ?? ''} /> @@ -1447,7 +1448,7 @@ {showSchemaExplorer} /> {:else if inputCat == 'ai-provider'} - + {:else if inputCat == 'email'} - +
diff --git a/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte b/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte index 6fa87abf64..a9706ecc75 100644 --- a/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte +++ b/frontend/src/lib/components/ChangeInstanceUsernameInner.svelte @@ -6,30 +6,34 @@ import { createEventDispatcher } from 'svelte' interface Props { - email: string; - username: string; - isConflict?: boolean; - noPadding?: boolean; + email: string + username: string + isConflict?: boolean + noPadding?: boolean } - let { - email, - username = $bindable(), - isConflict = false, - noPadding = false - }: Props = $props(); + let { email, username = $bindable(), isConflict = false, noPadding = false }: Props = $props() let loading = $state(false) - let usernameInfo: - | { - username: string - workspace_usernames: { - workspace_id: string - username: string - }[] - } - | undefined = $state(undefined) + type UsernameInfo = { + username: string + workspace_usernames: { + workspace_id: string + username: string + }[] + } + + let usernameInfo: UsernameInfo | undefined = $state(undefined) + + let affectedWorkspaces = $derived.by( + () => usernameInfo?.workspace_usernames.filter((w) => w.username !== username) ?? [] + ) + let isRenaming = $derived.by( + () => + usernameInfo !== undefined && + (username !== usernameInfo.username || affectedWorkspaces.length > 0) + ) function handleKeyUp(event: KeyboardEvent) { const key = event.key @@ -53,6 +57,11 @@ const dispatch = createEventDispatcher() async function renameUser() { + // Renaming before the current usernames are known would apply a change whose scope + // the "Manual action required" warning could not have been shown for. + if (!usernameInfo) { + return + } loading = true try { const automateUsernameCreation = @@ -102,31 +111,30 @@ Users are required to have an instance-wide username that is shared across all workspaces. However, this user has different usernames in different workspaces. - {#if usernameInfo?.workspace_usernames && usernameInfo.workspace_usernames.filter((w) => w.username !== username).length > 0} + {#if affectedWorkspaces.length > 0}

- Workspaces requiring username modification: {usernameInfo.workspace_usernames - .filter((w) => w.username !== username) + Workspaces requiring username modification: {affectedWorkspaces .map((wu) => `${wu.workspace_id} (${wu.username})`) .join(', ')} {/if} {/if} - {#if !isConflict && usernameInfo?.workspace_usernames && usernameInfo.workspace_usernames.filter((w) => w.username !== username).length > 0} + {#if !isConflict && affectedWorkspaces.length > 0} - {usernameInfo.workspace_usernames - .filter((w) => w.username !== username) - .map((wu) => `${wu.workspace_id}`) - .join(', ')} + {affectedWorkspaces.map((wu) => `${wu.workspace_id}`).join(', ')} {/if} - - This operation does not handle references in scripts, workflows and applications to scripts in - the workspace, and references in resources to variables. You will have to handle those manually. -
-
+ {#if isRenaming} + + This operation does not handle references in scripts, workflows and applications to scripts in + the workspace, and references in resources to variables. You will have to handle those + manually. +
+
+ {/if} +
diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte index b6833521f8..3bda6aa216 100644 --- a/frontend/src/lib/components/DdlMigrationGuard.svelte +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -3,14 +3,15 @@ import Modal2 from './common/modal/Modal2.svelte' import NewDataTableMigrationModal from './workspaceSettings/NewDataTableMigrationModal.svelte' import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' - import { splitSqlStatements, isDdlStatement } from './sqlDdl' + import { joinSqlStatements, splitSqlRuns } from './sqlDdl' + import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry' import { CornerDownLeft } from 'lucide-svelte' let { workspace, datatable }: { workspace: string; datatable: string } = $props() type Choice = 'run' | 'migrate' | 'cancel' - let promptStatement = $state(undefined) + let promptStatements = $state([]) let promptOpen = $state(false) let resolvePrompt: ((choice: Choice) => void) | undefined = undefined let resolveMigrationClosed: ((created: boolean) => void) | undefined = undefined @@ -22,11 +23,15 @@ // toast action after a migration is created here. let migrationsModal = $state(undefined) + // The block is shown as-is in the prompt and becomes the migration body, where + // every statement inside the BEGIN; ... END; frame must be `;`-terminated. + let promptSql = $derived(joinSqlStatements(promptStatements)) + function finishPrompt(choice: Choice) { const r = resolvePrompt resolvePrompt = undefined promptOpen = false - promptStatement = undefined + promptStatements = [] r?.(choice) } @@ -48,10 +53,10 @@ } } - function promptDdl(statement: string): Promise { + function promptDdl(statements: string[]): Promise { return new Promise((resolve) => { resolvePrompt = resolve - promptStatement = statement + promptStatements = statements promptOpen = true }) } @@ -65,61 +70,66 @@ // Open the prefilled new-migration modal. Resolves with whether a migration // was actually created (false if the user cancelled / closed it). - function openMigrationModal(statement: string): Promise { + function openMigrationModal(sql: string): Promise { return new Promise((resolve) => { resolveMigrationClosed = (created: boolean) => resolve(created) - newMigrationModal?.open({ codeUp: statement }) + newMigrationModal?.open({ codeUp: sql }) }) } /** - * Inspect `code` for DDL statements. For each one, prompt the user to run it - * anyway or turn it into a migration (prompts shown one at a time). Returns - * whether to proceed and the code to run (with migrated statements stripped). + * Inspect `code` for DDL statements. Each run of adjacent DDL statements is + * prompted for once (runs shown one at a time) and becomes a single migration, + * so a chain of schema changes applies in one transaction instead of asking + * once per statement. Returns whether to proceed and the code to run (with + * migrated statements stripped). */ export async function guard( code: string ): Promise<{ proceed: boolean; code: string; ranMigration: boolean }> { migrationRan = false - const statements = splitSqlStatements(code) - if (!statements.some((s) => isDdlStatement(s))) { + const runs = splitSqlRuns(code) + if (!runs.some((r) => r.isDdl)) { return { proceed: true, code, ranMigration: false } } const kept: string[] = [] - for (const statement of statements) { - if (!isDdlStatement(statement)) { - kept.push(statement) + for (const run of runs) { + if (!run.isDdl) { + kept.push(...run.statements) continue } - // Re-prompt for this statement until the user makes a terminal choice; + // Re-prompt for this run until the user makes a terminal choice; // cancelling the migration modal returns to the prompt with the DDL intact. for (;;) { - const choice = await promptDdl(statement) + const choice = await promptDdl(run.statements) if (choice === 'cancel') { + logDdlGuardChoice('cancelled') return { proceed: false, code, ranMigration: migrationRan } } if (choice === 'run') { - kept.push(statement) + logDdlGuardChoice('run_anyway') + kept.push(...run.statements) break } - // migrate: only strip the statement once a migration is actually + // migrate: only strip the statements once a migration is actually // created; if the modal was cancelled, loop back to the prompt. - const created = await openMigrationModal(statement) + const created = await openMigrationModal(joinSqlStatements(run.statements)) if (created) { + logDdlGuardChoice('migrated') break } } } - return { proceed: true, code: kept.join(';\n'), ranMigration: migrationRan } + return { proceed: true, code: joinSqlStatements(kept), ranMigration: migrationRan } } 1 ? 'Schema changes detected' : 'Schema change detected'} fixedWidth="md" fixedHeight="adaptive" bind:isOpen={promptOpen} @@ -127,12 +137,17 @@ >

- This looks like a schema-changing (DDL) statement. Schema changes are best tracked as - migrations rather than run ad-hoc. Create a migration for it instead? + {#if promptStatements.length > 1} + These {promptStatements.length} consecutive statements are schema-changing (DDL). Schema changes + are best tracked as migrations rather than run ad-hoc. Create a single migration for them instead? + {:else} + This looks like a schema-changing (DDL) statement. Schema changes are best tracked as + migrations rather than run ad-hoc. Create a migration for it instead? + {/if}

{promptStatement ?? ''}
{promptSql}
diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index c7e83b8a80..1bf9e7fde3 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -34,6 +34,7 @@ import FlowPreviewButtons from './flows/header/FlowPreviewButtons.svelte' import FlowModuleSchemaMap from './flows/map/FlowModuleSchemaMap.svelte' import FlowEditorPanel from './flows/content/FlowEditorPanel.svelte' + import AgentEditorModal from './flows/content/AgentEditorModal.svelte' import { deepEqual } from 'fast-equals' import { findModuleInFlow } from './flows/flowDiff' import { writable } from 'svelte/store' @@ -1244,7 +1245,6 @@
+ + true} /> {/if} diff --git a/frontend/src/lib/components/DiffDrawer.svelte b/frontend/src/lib/components/DiffDrawer.svelte index c1655da487..c09f5f31e3 100644 --- a/frontend/src/lib/components/DiffDrawer.svelte +++ b/frontend/src/lib/components/DiffDrawer.svelte @@ -7,10 +7,10 @@ import { cleanValueProperties, orderedJsonStringify, - orderedYamlStringify, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import type { Script } from '$lib/gen' import Select from './select/Select.svelte' import type { DiffVersionOption } from './diff_drawer' diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 8208fb193e..47bafa4d5a 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -32,6 +32,7 @@ import Alert from './common/alert/Alert.svelte' import AutoDataTable from './table/AutoDataTable.svelte' import Markdown from 'svelte-exmarkdown' + import { markdownProse } from './markdownProse' import Toggle from './Toggle.svelte' import FileDownload from './common/fileDownload/FileDownload.svelte' @@ -1229,7 +1230,7 @@
{:else if !forceJson && resultKind === 'markdown'} -
+
{:else if largeObject || hasBigInt} diff --git a/frontend/src/lib/components/DynamicInput.svelte b/frontend/src/lib/components/DynamicInput.svelte index 9f6f3b8261..76395f397f 100644 --- a/frontend/src/lib/components/DynamicInput.svelte +++ b/frontend/src/lib/components/DynamicInput.svelte @@ -35,6 +35,9 @@ name: string /** Workspace the helper script runs in; defaults to the nav workspace. */ workspace?: string + /** Reaches the fallback editor too, which is what renders when there is no + * `helperScript` — a caller disabling this argument means all of it. */ + disabled?: boolean } let { @@ -42,7 +45,8 @@ helperScript, format, otherArgs: otherArgs, - workspace = undefined + workspace = undefined, + disabled = false }: Props = $props() let [inputType, entrypoint] = $derived(format.includes('-') ? format.split('-', 2) : [format, '']) @@ -190,7 +194,7 @@ items={safeSelectItems(items || [])} placeholder="Select items" noItemsMsg={_items.status === 'loading' ? 'Loading...' : 'No items found'} - disabled={_items.status === 'loading'} + disabled={disabled || _items.status === 'loading'} /> {:else if inputType === 'dynselect'}
+ + Service name, environment, resource attributes, metrics temporality and other + options are set with environment variables. + {/if}
{:else if setting.fieldType == 'otel_tracing_proxy'} @@ -861,6 +870,8 @@ {:else if setting.fieldType == 'ws_connectivity'} + {:else if setting.fieldType == 'instance_banner'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index ce5323fe2b..f965362659 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -14,6 +14,7 @@ import { sleep } from '$lib/utils' import { enterpriseLicense } from '$lib/stores' + import { isCloudHosted } from '$lib/cloud' import { createEventDispatcher } from 'svelte' import { setLicense } from '$lib/enterpriseUtils' @@ -100,7 +101,8 @@ otel: {}, indexer_settings: {}, critical_error_channels: [], - github_enterprise_app: {} + github_enterprise_app: {}, + instance_banner: {} } function applyFormDefaults(vals: Record): void { @@ -524,6 +526,10 @@ for (const category of settingsKeys) { const categorySettings = getSettingsForCategory(category) result[category] = categorySettings.some((s) => { + // A field the build never renders must not be able to block Save: off-cloud its + // value is unreachable, so an invalid one (from config sync, say) would leave the + // category permanently unsaveable with nothing on screen to fix. + if (s.cloudonly && !isCloudHosted()) return false if (s.isValid && !s.isValid(currentValues?.[s.key])) return true if (s.validate) { const errors = s.validate(currentValues?.[s.key]) @@ -1059,7 +1065,10 @@
  • instance base URL
  • login type usage (login type, count)
  • worker usage (worker, worker instance, vCPUs, memory)
  • -
  • user usage (author count, operator count)
  • +
  • user usage (author count, operator count, the distinct guests of the last 30 days, + the seats they add past the free allowance, and the workspaces that allow guests)
  • superadmin email addresses
  • development instance status
  • @@ -1071,12 +1080,19 @@ >feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions are started for, whether AI chat skills are turned on or off and how often one is - loaded, and the plan tier and quota shown when the execution meter is opened, last 30 - days)
  • feature adoption (counts of which flow, script, trigger and worker features your - deployed items use)
  • feature adoption (counts of which flow, script, trigger, worker and data table + features your deployed items use, including how many apps run sandboxed, how many data + tables exist per database kind, how many use migrations, and what references them)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code @@ -1119,18 +1135,28 @@
  • job usage (language, total duration, count)
  • login type usage (login type, count)
  • worker usage (worker, worker instance, vCPUs, memory)
  • -
  • user usage (author count, operator count)
  • +
  • user usage (author count, operator count, the distinct guests of the last 30 days, + the seats they add past the free allowance, and the workspaces that allow guests)
  • development instance status
  • feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions are started for, whether AI chat skills are turned on or off and how often one is - loaded, and the plan tier and quota shown when the execution meter is opened, last 30 - days)
  • feature adoption (counts of which flow, script, trigger and worker features your - deployed items use)
  • feature adoption (counts of which flow, script, trigger, worker and data table + features your deployed items use, including how many apps run sandboxed, how many data + tables exist per database kind, how many use migrations, and what references them)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index eb6cc9bc97..64ca912608 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -21,9 +21,18 @@ args: any argLabel?: string | undefined workspace?: string | undefined + /** Drop the header's expand-into-a-drawer button, for a caller that already offers a + * way to open the run in full. */ + disableExpand?: boolean } - let { id = undefined, args, argLabel = undefined, workspace = undefined }: Props = $props() + let { + id = undefined, + args, + argLabel = undefined, + workspace = undefined, + disableExpand = false + }: Props = $props() // Internal flag injected by "test this step" runs to suppress the asset // dispatcher. Not a real input: shown as a badge instead of a table row, @@ -125,18 +134,20 @@ ${Object.entries(displayArgs) Value {#snippet headerAction()} -
    - -
    + {#if !disableExpand} +
    + +
    + {/if} {/snippet} diff --git a/frontend/src/lib/components/LocalDraftBanner.svelte b/frontend/src/lib/components/LocalDraftBanner.svelte index dff131b281..af2787222e 100644 --- a/frontend/src/lib/components/LocalDraftBanner.svelte +++ b/frontend/src/lib/components/LocalDraftBanner.svelte @@ -2,12 +2,8 @@ import { Button } from '$lib/components/common' import DiffDrawer from '$lib/components/DiffDrawer.svelte' import { classes } from '$lib/components/common/alert/model' - import { - cleanValueProperties, - orderedYamlStringify, - replaceFalseWithUndefined, - type Value - } from '$lib/utils' + import { cleanValueProperties, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AlertCircle, Diff } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' import { fade } from 'svelte/transition' diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 10f996a33f..e4b73478aa 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -16,15 +16,13 @@
    @@ -31,9 +32,11 @@
    -
    - - -
    + {#if showVersion} +
    + + +
    + {/if}
    diff --git a/frontend/src/lib/components/ModulePreviewForm.svelte b/frontend/src/lib/components/ModulePreviewForm.svelte index 88b99a05c5..72bb9fe36b 100644 --- a/frontend/src/lib/components/ModulePreviewForm.svelte +++ b/frontend/src/lib/components/ModulePreviewForm.svelte @@ -14,6 +14,7 @@ import { getResourceTypes } from './resourceTypesStore' import { twMerge } from 'tailwind-merge' import { workspaceStore } from '$lib/stores' + import { AGENT_FIELDS, initialVisibleAgentFields } from './flows/agentFormFields' interface Props { schema: Schema | { properties?: Record; required?: string[] } @@ -43,15 +44,45 @@ isValid = allTrue(inputCheck) ?? false }) + /** An agent asks for the same fields here that its own form shows: a setting the step leaves + * unset is not something a run needs told, and listing all eleven buries the message under the + * configuration. What the step configures stays, as it does on any other step. A schema key the + * field registry doesn't know is kept, so a new one is never silently dropped. A run input is + * kept whatever the step holds: this form has no add-field control, so hiding one would leave + * no way at all to supply it. */ + let schemaKeys = $derived(Object.keys(schema?.properties ?? {})) + + let visibleKeys = $derived.by(() => { + const all = schemaKeys + if ((mod.value as { type?: string })?.type !== 'aiagent') return all + const transforms = (mod.value as { input_transforms?: Record }) + ?.input_transforms + const visible = initialVisibleAgentFields(transforms, schema?.properties) + const known = new Set(AGENT_FIELDS.filter((f) => !f.runInput).map((f) => f.key)) + return all.filter((key) => !known.has(key) || visible.has(key)) + }) + let keys: string[] = $state([]) $effect(() => { - let lkeys = Object.keys(schema?.properties ?? {}) + let lkeys = visibleKeys if (schema?.properties && JSON.stringify(lkeys) != JSON.stringify(keys)) { keys = lkeys - untrack(() => stepsInputArgs?.removeExtraKey(mod.id, keys)) + // Pruned against the schema rather than against what is shown. What a run was given for a + // field lives only here, so dropping it when the field merely stops being displayed would + // discard it: an agent hides the settings its step leaves unset, and clearing one in the + // Inputs tab hides it. + untrack(() => stepsInputArgs?.removeExtraKey(mod.id, schemaKeys)) } }) + /** Whether re-evaluating has anything to restore. A field the step configures nothing for + * evaluates to blank, so the control would only clear what was typed to run with. */ + function hasConfiguredInput(argName: string): boolean { + const transform = (mod.value as any)?.input_transforms?.[argName] + if (!transform) return false + return transform.type === 'javascript' ? !!transform.expr : transform.value !== undefined + } + function plugIt(argName: string) { stepsInputArgs?.setEvaluatedStepArg( mod.id, @@ -158,7 +189,7 @@ workspace={opWs} > {#snippet fieldHeaderActions()} - {#if stepsInputArgs?.isArgManuallySet(mod.id, argName)} + {#if stepsInputArgs?.isArgManuallySet(mod.id, argName) && hasConfiguredInput(argName)}
  • - {/each} +{#if options.length > 0} +
    + {#each options as option (option)} + + {/each} +
    + Custom scopes {/if} +{#each custom as v, i (i)} +
    + setRow(i, e.currentTarget.value) }} + /> +
    +{/each} +
    - - ({(scopes ?? []).length} item{(scopes ?? []).length > 1 ? 's' : ''}) - + {#if custom.length > 0} + + ({custom.length} item{custom.length > 1 ? 's' : ''}) + + {/if}
    diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index 14658a4228..9ddd7f7fdc 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -1,3 +1,42 @@ + +
    - {#if loading} + {#snippet action()} + + {#snippet children({ item })} + {#each Object.keys(WINDOWS) as key (key)} + + {/each} + {/snippet} + + {/snippet} + + {#if metrics.error} + {metrics.error.message} + {:else if metrics.current === undefined} - {:else if noMetrics} -

    No jobs delayed by more than 3 seconds in the last 14 days

    + {:else if metrics.current.tags.length === 0} +

    + No jobs delayed by more than 3 seconds in the last {WINDOWS[windowKey].label} +

    {:else}
    - {#if countData} - 3s)' - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - y: { - title: { - display: true, - text: 'count' - } - } - } - }} - /> - {/if} - {#if delayData} - 3s)' - }, - tooltip: { - callbacks: { - label: function (context) { - // @ts-ignore - if (context.raw.y === 1) { - return context.dataset.label + ': 0' - } else { - // @ts-ignore - return context.dataset.label + ': ' + context.raw.y - } - } - } - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - - y: { - type: 'logarithmic', - title: { - display: true, - text: 'delay (s)' - }, - ticks: { - callback: (value, _) => (value === 1 ? '0' : value) - } - } - } - }} - /> - {/if} + + - Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are - included in the graph. + Only tags with jobs delayed by more than 3 seconds in this window are included. At wide + windows a line shows the highest value of each time slot, so short spikes stay visible.
    {/if} diff --git a/frontend/src/lib/components/QueueStatusTable.svelte b/frontend/src/lib/components/QueueStatusTable.svelte new file mode 100644 index 0000000000..5f34e80c60 --- /dev/null +++ b/frontend/src/lib/components/QueueStatusTable.svelte @@ -0,0 +1,115 @@ + + +
    + {#snippet action()} +

    + Tag + Waiting + Next job's wait + Running + Workers + + + + {#each rows as s (s.tag)} + + {s.tag} + {s.waiting} + + {s.delay === undefined ? '-' : msToReadableTime(s.delay * 1000, 0)} + + {s.running} + + {#if unserved(s)} + + + No worker currently pulls this tag + + {:else} + {s.workers} + {/if} + + + {/each} + + + {/if} + diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 367888655f..bcb886ee7d 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -1,19 +1,27 @@ @@ -382,7 +537,28 @@ {/if} - {#if current} + {#if holdsCredential && selected} + +
    +
    + The URL carries no credential. Windmill stores the token and renews it before it + expires. + {#if urlDirty} + Save your URL change to replace the token. + {/if} +
    + +
    +
    + {/if} + + + {#if current && can_write !== undefined} {#key current} current!.path, setPath} @@ -391,7 +567,7 @@ bind:args={current.args} bind:wsSpecific={current.wsSpecific} bind:isValid - bind:viewJsonSchema + bind:viewJsonSchema={() => viewJsonSchema ?? false, (v) => (viewJsonSchema = v)} bind:jsonError {initialPath} {hidePath} @@ -404,6 +580,7 @@ {resourceToEdit} onLoadResourceType={() => resourceTypeResource.refetch()} workspace={selected} + actingUser={acting.in(selected) ?? null} /> {/key} {/if} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 296ab693a8..36d49d9149 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -5,8 +5,9 @@ import { History, Loader2, Save } from 'lucide-svelte' import WsSpecificVersions from './WsSpecificVersions.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { workspaceStore } from '$lib/stores' import { isOwner } from '$lib/utils' + import { useActingUser } from '$lib/actingUser.svelte' import LocalDraftBanner from './LocalDraftBanner.svelte' import OpenInSessionButton from './sessions/OpenInSessionButton.svelte' import { @@ -25,6 +26,8 @@ onRestored = undefined, onSaved = undefined }: { + /** Workspace this drawer acts in. Optional; the navigation workspace is substituted + * once, at `effectiveWorkspace`, and nowhere else in this file. */ workspace?: string disableChatOffset?: boolean onRestored?: () => void @@ -52,23 +55,44 @@ let path: string | undefined = $state(undefined) let selected: string | undefined = $state(undefined) + let viewJsonSchema = $state(false) let effectiveWorkspace = $derived(workspace ?? $workspaceStore!) // The editor renders whichever workspace-specific variant `selected` points at, so history has // to follow it too — otherwise a restore would write over the variant the user is not looking at. let historyWorkspace = $derived(selected ?? effectiveWorkspace) - // Clearing is irreversible and the backend gates it on ownership, not write access. $userStore - // describes the user in the workspace they are signed into, so it can only answer for that one: - // history pointed anywhere else — a ws-specific variant, or an explicit `workspace` prop — gets - // no Clear button rather than a verdict computed from the wrong membership. - let canClearSelected = $derived( - historyWorkspace === $workspaceStore && isOwner(path ?? '', $userStore, $workspaceStore) - ) + // Gated on `path`: this drawer outlives every resource it opens, so there is nothing to + // answer about until one is open. + const historyUser = useActingUser(() => (path ? historyWorkspace : undefined)) + // Clearing is irreversible and the backend gates it on ownership, not write access, so the + // verdict has to come from the membership `historyWorkspace` knows about. An unresolved + // user gets no Clear button rather than one computed from another workspace's rights. + let canClearSelected = $derived(isOwner(path ?? '', historyUser.current, historyWorkspace)) - export async function initEdit(p: string): Promise { + // A close reaches `on:close` on a later flush, by which point a caller that closed this drawer to + // open another editor has already anchored the new one. Clearing then would strip that anchor. + let keepAnchorOnClose = false + + /** Shut this drawer without going through its own close button, for a caller opening the other + * editor over the same list. `keepAnchor` when that caller anchors what it opens instead. */ + export function close(opts?: { keepAnchor?: boolean }): void { + keepAnchorOnClose = opts?.keepAnchor ?? false + drawer?.closeDrawer?.() + } + + /** `json` opens on the JSON editor instead of the resource type's form. For a type with a + * dedicated editor elsewhere: the generic form would render its configuration field by field, + * and materialize a default into every one the value leaves out. */ + export async function initEdit(p: string, opts?: { json?: boolean }): Promise { + // A `close({ keepAnchor })` on an already-closed drawer emits no close event, so the flag + // would still be standing when the next drawer session ends and would swallow that one's + // anchor clear. Every session starts having to clear its own. + keepAnchorOnClose = false + historyUser.forgetFailures() resource_type = undefined path = p selected = effectiveWorkspace + viewJsonSchema = opts?.json ?? false drawer?.openDrawer?.() setPageDrawerAnchor(RESOURCES_PATH, p) } @@ -77,10 +101,16 @@ resourceType: string, nDefaultValues?: Record ): Promise { + keepAnchorOnClose = false + historyUser.forgetFailures() path = undefined resource_type = resourceType defaultValues = nDefaultValues selected = effectiveWorkspace + // This drawer outlives what it opens on, so the view has to be set by every entry point + // rather than left where the last one put it: a new resource is a typed form, whoever was + // looking at JSON before. + viewJsonSchema = false drawer?.openDrawer?.() } @@ -97,7 +127,13 @@ bind:this={drawer} size="50rem" {disableChatOffset} - on:close={() => clearPageDrawerAnchor(RESOURCES_PATH)} + on:close={() => { + if (keepAnchorOnClose) { + keepAnchorOnClose = false + return + } + clearPageDrawerAnchor(RESOURCES_PATH) + }} > (hasLocalDraft = v)} onCanWriteChange={(v) => (canWriteSelected = v)} /> diff --git a/frontend/src/lib/components/ResourceForm.svelte b/frontend/src/lib/components/ResourceForm.svelte index cddd83aae7..df1aa5bd99 100644 --- a/frontend/src/lib/components/ResourceForm.svelte +++ b/frontend/src/lib/components/ResourceForm.svelte @@ -8,7 +8,7 @@ import Path from './Path.svelte' import LabelsInput from './LabelsInput.svelte' import Required from './Required.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { workspaceStore, type UserExt } from '$lib/stores' import SchemaForm from './SchemaForm.svelte' import SimpleEditor from './SimpleEditor.svelte' import FilesetEditor from './FilesetEditor.svelte' @@ -19,6 +19,7 @@ import GfmMarkdown from './GfmMarkdown.svelte' import TestTriggerConnection from './triggers/TestTriggerConnection.svelte' import GitHubAppIntegration from './GitHubAppIntegration.svelte' + import GitLabIntegration from './GitLabIntegration.svelte' import Button from './common/button/Button.svelte' import ResourceGen from './copilot/ResourceGen.svelte' import SyncResourceTypes from './SyncResourceTypes.svelte' @@ -37,7 +38,9 @@ viewJsonSchema: boolean jsonError: string deployTo: string | undefined - can_write: boolean + /** `undefined` while the acting user or the resource is still being resolved: neither a + * grant nor the denial the read-only alert announces. */ + can_write: boolean | undefined resource_type: string | undefined resourceTypeInfo: ResourceType | undefined resourceSchema: Schema | undefined @@ -47,6 +50,14 @@ /** Workspace the path is validated against and the connection is tested in; * defaults to the nav workspace. */ workspace?: string | undefined + /** The user acting in `workspace`, resolved by the editor above. `undefined` while + * `null` while that lookup is pending or after it failed: every check below then + * refuses, rather than answering with the navigation user's rights in another + * workspace. */ + actingUser: UserExt | null + /** Fired once the GitLab picker has stored the picked project's token, so a + * form that would otherwise file the URL as a secret knows it holds none. */ + onCredentialStored?: () => void } let { @@ -68,7 +79,9 @@ loadingSchema, resourceToEdit, onLoadResourceType, - workspace = undefined + workspace = undefined, + actingUser, + onCredentialStored }: Props = $props() let ws = $derived(workspace ?? $workspaceStore) @@ -148,7 +161,7 @@ {#if !hidePath}
    - {#if !can_write} + {#if can_write === false}
    You only have read access to this resource and cannot edit it @@ -158,12 +171,13 @@
    @@ -241,19 +255,37 @@ workspaceOverride={workspace} /> {/if} - {#if resource_type === 'git_repository' && $workspaceStore && ($userStore?.is_admin || $userStore?.is_super_admin)} + {#if resource_type === 'git_repository' && ws && (actingUser?.is_admin || actingUser?.is_super_admin)} { args = newArgs - if (viewJsonSchema) { + // The raw editor is also what a workspace missing the resource type + // gets, and it holds its own copy of the value: without this the + // picker fills in a URL nothing on screen ever shows. + if (viewJsonSchema || !resourceSchema) { rawCode = JSON.stringify(args, null, 2) } }} onDescriptionUpdate={(newDescription) => (description = newDescription)} /> + { + args = newArgs + // The raw editor is also what a workspace missing the resource type + // gets, and it holds its own copy of the value: without this the + // picker fills in a URL nothing on screen ever shows. + if (viewJsonSchema || !resourceSchema) { + rawCode = JSON.stringify(args, null, 2) + } + }} + /> {/if}
    diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index 8342bbecf6..06023b1bb4 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -33,6 +33,10 @@ datatableAsPgResource?: boolean workspace?: string | undefined disableChatOffset?: boolean + /** Fires when this picker sets a resource, with the type it carries, and with `undefined` on + * clear. Unlike an effect on `valueType` it never fires for the value the picker was opened + * on, but `selectFirst` choosing the only candidate during a load does count as setting one. */ + onValueChange?: (path: string | undefined, type: string | undefined) => void } let { @@ -54,7 +58,8 @@ excludedValues = undefined, datatableAsPgResource = false, workspace = undefined, - disableChatOffset = false + disableChatOffset = false, + onValueChange = undefined }: Props = $props() let effectiveWorkspace = $derived(workspace ?? $workspaceStore!) @@ -155,6 +160,7 @@ if (collection.length == 1 && selectFirst && (value == undefined || value == '')) { value = collection[0].value valueType = collection[0].type + onValueChange?.(value, valueType) } } catch (e) { sendUserToast('Failed to load resource types', true) @@ -196,6 +202,7 @@ await loadResources(resourceType) value = e.detail valueType = collection.find((x) => x?.value == value)?.type + onValueChange?.(value, valueType) }} bind:this={appConnect} {expressOAuthSetup} @@ -211,6 +218,7 @@ if (e.detail) { value = e.detail valueType = collection.find((x) => x?.value == value)?.type + onValueChange?.(value, valueType) // valueSelect = { value: e.detail, label: e.detail, type: valueType ?? '' } } }} @@ -237,12 +245,14 @@ (v) => { value = v valueType = collection.find((x) => x?.value == v)?.type + onValueChange?.(value, valueType) } } onClear={() => { initialValue = undefined value = undefined valueType = undefined + onValueChange?.(undefined, undefined) onClear?.() }} items={collection} @@ -281,6 +291,7 @@
    {#if resourceType?.includes(',')} ({ displayName: `${rt} resource`, @@ -294,7 +305,7 @@ color="light" variant="contained" wrapperClasses="flex-1" - btnClasses="rounded-none mt-0.5" + btnClasses="rounded-none" size="sm" startIcon={{ icon: Plus }} > diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index a371f3eea9..510f774587 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -24,6 +24,7 @@ import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import { untrack } from 'svelte' import { processSecretArgs } from './secretArgUtils' + import { enforceDisabledDefaults, resetKeysToast } from './job_args' import PowerShellCommonParams from './PowerShellCommonParams.svelte' let reloadArgs = $state(0) @@ -60,11 +61,12 @@ export async function run(overrideScheduledForStr?: string | undefined | null) { let processedArgs: Record + const { args: withDefaults, resetKeys } = enforceDisabledDefaults(args ?? {}, runnable?.schema) + if (resetKeys.length > 0) { + sendUserToast(resetKeysToast(resetKeys)) + } try { - processedArgs = await processSecretArgs( - enforceDisabledDefaults(args ?? {}, true), - runnable?.schema - ) + processedArgs = await processSecretArgs(withDefaults, runnable?.schema) } catch (e) { sendUserToast('Failed to process sensitive args: ' + e, true) return @@ -178,30 +180,6 @@ } } - function enforceDisabledDefaults( - args: Record, - notify: boolean = false - ): Record { - const schema = runnable?.schema - if (!schema?.properties) return args - const result = { ...args } - const resetKeys: string[] = [] - for (const [key, prop] of Object.entries(schema.properties) as [string, any][]) { - if (prop?.disabled && 'default' in prop) { - if (notify && result[key] !== prop.default) { - resetKeys.push(key) - } - result[key] = prop.default - } - } - if (resetKeys.length > 0) { - sendUserToast( - `Disabled field${resetKeys.length > 1 ? 's' : ''} ${resetKeys.map((k) => `'${k}'`).join(', ')} reset to default value${resetKeys.length > 1 ? 's' : ''}` - ) - } - return result - } - /** Rewrite the open JSON editor from the current args. Only for args replaced from outside * the editor: entering the JSON view already starts from whatever `args` holds. */ export function syncJsonEditor() { @@ -322,7 +300,7 @@ bind:this={jsonEditor} on:select={(e) => { if (e.detail) { - args = enforceDisabledDefaults(e.detail) + args = enforceDisabledDefaults(e.detail, runnable?.schema).args } }} initialCode={argsToJsonPayload(runnable.schema, args)} diff --git a/frontend/src/lib/components/RunPageTutorials.svelte b/frontend/src/lib/components/RunPageTutorials.svelte deleted file mode 100644 index e46b1c9019..0000000000 --- a/frontend/src/lib/components/RunPageTutorials.svelte +++ /dev/null @@ -1,25 +0,0 @@ - - - diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 03b410d232..cac8952706 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -84,6 +84,8 @@ initialPath?: string } + let { initialPath }: Props = $props() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -100,25 +102,30 @@ let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean') let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean') - let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema)) + function filterSeeds() { + return { + path: initialPath || undefined, + job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined, + show_future_jobs: showFutureJobsStorage.val === false ? false : undefined + } + } - let { initialPath }: Props = $props() + let filters = useUrlSyncedFilterInstance( + untrack(() => runsFilterSearchbarSchema), + untrack(filterSeeds) + ) + + // `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own + // "Runs" entry, `/runs/` → `/runs`, Back — rewrites the query without remounting, and + // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, + // which never reaches `page.url`, so a filter the user clears stays cleared. + $effect(() => { + page.url.href + untrack(() => filters.seed(filterSeeds())) + }) let batchRerunOptionsIsOpen = $state(false) - // Initialize path filter from route param if provided and not already set via query params - if (untrack(() => initialPath) && !filters.val.path) { - filters.val.path = untrack(() => initialPath) - } - - // Apply persistent toggle values from local storage if URL doesn't specify them - if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) { - filters.val.job_trigger_kind = '!schedule' - } - if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) { - filters.val.show_future_jobs = false - } - // Sync toggle state back to local storage when filters change $effect(() => { if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') { diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 1d7bade71a..c05a69d1d9 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -102,7 +102,7 @@ import type { SavedAndModifiedValue } from './common/confirmationModal/unsavedTypes' import DeployButton from './DeployButton.svelte' import { type Trigger, deployTriggers, handleSelectTriggerFromKind } from './triggers/utils' - import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' + import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte' import { Triggers } from './triggers/triggers.svelte' import type { ScriptBuilderProps } from './script_builder' import WorkerTagSelect from './WorkerTagSelect.svelte' @@ -1215,7 +1215,7 @@ currentValue={script} /> - t.draftConfig)} on:canceled={() => { diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index fdab141d03..f8028aeb1f 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -15,7 +15,7 @@ import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils' import Editor from './Editor.svelte' import { inferArgs, inferAssets, inferAnsibleExecutionMode } from '$lib/infer' - import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + import { injectPartitionArg } from '$lib/scriptEditorSchema' import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow' import WacDiagram from '$lib/components/graph/WacDiagram.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -1074,55 +1074,6 @@ } } - // A `// partitioned` pipeline script is materialized one slice at a time and - // receives the slice as a runtime `partition` arg (the cascade injects it in - // production). It isn't a code parameter, so schema inference doesn't see it — - // surface it in the test form so a partitioned script can be run manually. - function injectPartitionArg( - s: any, - a: Record | undefined, - l: string | undefined, - c: string - ) { - try { - if (l !== 'duckdb' || !s?.properties) return - const part = parsePipelineAnnotations(c).partition - if (!part) return - // Date-based partition kinds render a date / datetime picker; a dynamic - // key is a free-form string. - const format = - part.kind === 'hourly' - ? 'date-time' - : part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly' - ? 'date' - : undefined - if (!s.properties['partition']) { - s.properties['partition'] = { - type: 'string', - ...(format ? { format } : {}), - // ISO output so partition keys sort lexicographically (the date - // picker defaults to dd-MM-yyyy otherwise). - ...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}), - description: - part.kind === 'dynamic' - ? 'Partition key value to materialize.' - : `Partition (${part.kind}) to materialize.` - } - if (Array.isArray(s.order) && !s.order.includes('partition')) { - s.order = ['partition', ...s.order] - } - } - // Pre-fill the *test* arg with the current slice for date kinds — a - // convenience default, kept on the args (not baked into the schema, - // where it would persist to the deployed script and go stale). - if (format && a && (a['partition'] == null || a['partition'] === '')) { - const now = new Date() - a['partition'] = - format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16) - } - } catch (e) {} - } - async function inferModuleSchema() { if (activeModuleTab === null) return try { diff --git a/frontend/src/lib/components/Section.svelte b/frontend/src/lib/components/Section.svelte index f1425c51b7..badc7c9531 100644 --- a/frontend/src/lib/components/Section.svelte +++ b/frontend/src/lib/components/Section.svelte @@ -102,7 +102,7 @@ transition:slide={animate || collapsable ? { duration: 200 } : { duration: 0 }} > {#if description} -
    {@html description}
    +
    {@html description}
    {/if}
    diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 65713e19f7..8133494f18 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -63,6 +63,10 @@ /** Trailing debounce window (ms) on Monaco's onDidChangeModelContent. */ const CHANGE_TIMEOUT = 200 + /** Gap between the line numbers and the first character. Zero puts them flush, + * so a two-digit line reads as one token with the code. */ + const LINE_DECORATIONS_WIDTH = 6 + let changeTimeoutId: number | undefined = undefined // Monaco fires onDidChangeModelContent synchronously from within `setValue`, so without // this an authoritative overwrite reads as a user edit on the `input` event. @@ -112,7 +116,8 @@ minHeight = 1000, renderLineHighlight = 'none', suggestion, - leadingChangeSync = false + leadingChangeSync = false, + lineNumbersMinChars = 3 }: { lang: string code?: string @@ -149,6 +154,9 @@ * `code`; leave it off where each extra sync costs work downstream (an app * code input feeding an autoRefresh runnable re-runs a job per sync). */ leadingChangeSync?: boolean + /** Width of the line-number gutter, in characters. Same name, and same + * default, as `Editor`, so the two render line numbers alike. */ + lineNumbersMinChars?: number } = $props() let yPadding = MONACO_Y_PADDING @@ -312,10 +320,12 @@ if (model.getLanguageId() !== lang) { const currentCode = model.getValue() const uri = `file:///${hash}.${langToExt(lang)}` - const oldModel = model - const newModel = meditor.createModel(currentCode, lang, mUri.parse(uri)) - editor?.setModel(newModel) - oldModel.dispose() + // The old model goes first: `langToExt` maps anything it does not know to + // `unknown`, so the new uri is usually the one this model already holds, + // and creating over an occupied uri throws ("model already exists"). + editor?.setModel(null) + model.dispose() + editor?.setModel(meditor.createModel(currentCode, lang, mUri.parse(uri))) } // Update editor options for suggestions, validation decorations, and line numbers @@ -334,8 +344,8 @@ snippetsPreventQuickSuggestions: disableSuggestions }, lineNumbers: hideLineNumbers ? 'off' : 'on', - lineDecorationsWidth: hideLineNumbers ? 0 : 6, - lineNumbersMinChars: hideLineNumbers ? 0 : 2, + lineDecorationsWidth: hideLineNumbers ? 0 : LINE_DECORATIONS_WIDTH, + lineNumbersMinChars: hideLineNumbers ? 0 : lineNumbersMinChars, // Hide validation squiggles and decorations renderValidationDecorations: disableLinting ? 'off' : 'on', // Hide the validation margin indicators @@ -397,8 +407,11 @@ ...(yPadding !== undefined ? { padding: { bottom: yPadding, top: yPadding } } : {}), readOnly, renderLineHighlight, - lineDecorationsWidth: 0, - lineNumbersMinChars: 2, + // Same conditional as `updateModelAndOptions`: created correct rather than + // created wide and narrowed a tick later, which a caller hiding the gutter + // would see as a flash of indent. + lineDecorationsWidth: hideLineNumbers ? 0 : LINE_DECORATIONS_WIDTH, + lineNumbersMinChars: hideLineNumbers ? 0 : lineNumbersMinChars, fontSize: fontSize, quickSuggestions: disableSuggestions ? { other: false, comments: false, strings: false } diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index e74154b406..97b15fe443 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -1,5 +1,6 @@ - - diff --git a/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte b/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte deleted file mode 100644 index 61d88819e7..0000000000 --- a/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte +++ /dev/null @@ -1,64 +0,0 @@ - - - - - {#snippet titleBadge()} - Beta - {/snippet} -
    - {#if agentPath} - - {#key `${opWorkspace ?? ''}:${agentPath}`} - - {/key} - {:else} -
    - Evals run against a saved agent - - This agent is written into the flow step rather than saved as its own agent, so there is - nothing for a dataset and its runs to belong to. Save it as a reusable agent from the - step, and its evals start there. - -
    - {/if} -
    -
    diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte index 55792ba8e2..568d4302cd 100644 --- a/frontend/src/lib/components/aiEvals/EvalsPane.svelte +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -64,7 +64,8 @@ agentPath, opWorkspace = undefined, editedConfig = undefined, - location = $bindable() + location = $bindable(), + active = true }: { /** The agent under test. A dataset and its runs belong to an agent. */ agentPath: string @@ -77,6 +78,10 @@ /** The level the pane is on and the way out of it, reported up so the surface holding it * can put both in its header. Undefined at the root, which that surface already names. */ location?: EvalsLocation + /** False while the pane is parked off screen by a surface that keeps it mounted. Its own + * pages answer the arrow keys at `window`, which a parked instance would take from + * whatever is actually on screen. */ + active?: boolean } = $props() let ws = $derived(opWorkspace ?? $workspaceStore) @@ -666,19 +671,21 @@ warm class="grow min-h-0" current={!viewingRun || !loaded ? 'list' : 'run'} - onNavigate={(key) => { - // Right opens the run under the highlight, falling back to whichever was open before; - // left is the way back, the same as the breadcrumb. - if (key === 'run') { - // Both branches go through `openRun`: it is what brings the run's own dataset back, - // and the fallback run may be of a dataset the list has since moved off. - const id = highlightedRunId ?? experimentId - if (id) openRun(id) - } else if (key === 'list') { - viewingRun = false - selectedCaseId = undefined - } - }} + onNavigate={!active + ? undefined + : (key) => { + // Right opens the run under the highlight, falling back to whichever was open before; + // left is the way back, the same as the breadcrumb. + if (key === 'run') { + // Both branches go through `openRun`: it is what brings the run's own dataset back, + // and the fallback run may be of a dataset the list has since moved off. + const id = highlightedRunId ?? experimentId + if (id) openRun(id) + } else if (key === 'list') { + viewingRun = false + selectedCaseId = undefined + } + }} pages={[ { key: 'list', content: listPage }, { key: 'run', content: runPage } @@ -727,7 +734,7 @@ {datasets} {caseProgress} {loaded} - active={!viewingRun} + active={active && !viewingRun} {deployedHash} {currentVersion} onOpen={(e) => openRun(e.id)} diff --git a/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte new file mode 100644 index 0000000000..a1a58a4ce5 --- /dev/null +++ b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte @@ -0,0 +1,219 @@ + + +
    +
    +
    + {#if context} + + {context} + + {/if} + {#if job} +

    + Requested by {job.created_by} · +

    + {/if} +
    + {STATUS_BADGE[status].label} +
    + + {#if typeof approvalInfo.description === 'string'} +

    {approvalInfo.description}

    + {:else if approvalInfo.description != undefined} + + {/if} + + {#if status === 'pending'} + {#if hasForm} + {#if emptyString($enterpriseLicense)} + + {:else} + + {/if} + {/if} + + {#if approvalInfo.can_approve} +
    + {#if approvalInfo.hide_cancel !== true} + + {:else} +
    + {/if} + +
    + {#if isSelfApprovalBypass} + + As an administrator, by approving or rejecting this request, you bypass the self-approval + interdiction. + + {/if} + {:else if approvalInfo.user_auth_required && !$userStore} +

    Sign in to review this request.

    + + {:else} +
    +

    You are not authorized to approve this request.

    + {#if isSelfApprovalRefused} +

    Self-approval is disabled for this step.

    + {/if} + {#if groupsRequired.length > 0} +

    + Only members of the following groups can approve: + {groupsRequired.join(', ')} +

    + {/if} +
    + {/if} + {:else} +
    + {#if status === 'approved'} + + {:else if status === 'rejected'} + + {:else} + + {/if} +
    + + {status === 'closed' ? 'This request is closed' : STATUS_BADGE[status].label} + + + {#if status === 'approved'} + Your approval was recorded. You can close this page. + {:else if status === 'rejected'} + Your rejection was recorded. You can close this page. + {:else} + The flow is no longer waiting for approval. + {/if} + +
    +
    + {/if} + + {#if !isLocked && ((status === 'pending' && approvalInfo.approvers.length > 0) || isWorkspaceMember)} +
    + + {#if status === 'pending' && approvalInfo.approvers.length > 0} + Already approved by {approvalInfo.approvers.map((a) => a.approver).join(', ')} + {/if} + + {#if isWorkspaceMember} + + {/if} +
    + {/if} +
    diff --git a/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts new file mode 100644 index 0000000000..1dec5e9da8 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { renderDbEqualityFilter, renderDbLiteral } from './utils' + +describe('renderDbLiteral', () => { + it('doubles single quotes on every dialect', () => { + expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'") + expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'") + }) + + it('doubles backslashes only where the dialect treats them as escapes', () => { + expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'") + expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'") + expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'") + }) + + it('marks SQL Server strings as Unicode constants', () => { + expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'") + }) + + it('renders numbers and booleans without quotes', () => { + expect(renderDbLiteral(42, 'postgresql')).toBe('42') + expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE') + expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1') + }) + + it('has no literal for values that cannot be compared safely', () => { + expect(renderDbLiteral(null, 'postgresql')).toBeUndefined() + expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined() + expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined() + }) +}) + +describe('renderDbEqualityFilter', () => { + it('quotes the identifier per dialect', () => { + expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'") + expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined() + }) + + it('doubles a delimiter embedded in the identifier', () => { + expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`) + expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1') + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index bf81ec07bb..c0477e8408 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -333,25 +333,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string { .join(', ') } +/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an + * embedded delimiter. */ export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string { switch (dbType) { case 'postgresql': - return `"${identifier}"` // PostgreSQL uses double quotes for identifiers - case 'ms_sql_server': - return `[${identifier}]` // MSSQL uses square brackets for identifiers - case 'mysql': - return `\`${identifier}\`` // MySQL uses backticks case 'snowflake': - return `"${identifier}"` // Snowflake uses double quotes for identifiers - case 'bigquery': - return `\`${identifier}\`` // BigQuery uses backticks case 'duckdb': - return `"${identifier}"` // DuckDB uses double quotes for identifiers + return `"${identifier.replace(/"/g, '""')}"` + case 'ms_sql_server': + return `[${identifier.replace(/]/g, ']]')}]` + case 'mysql': + case 'bigquery': + return `\`${identifier.replace(/`/g, '``')}\`` default: throw new Error('Unsupported database type: ' + dbType) } } +/** Renders a cell value as a SQL literal. Returns undefined for values that + * have no safe literal form (null, objects, non-finite numbers). */ +export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined { + if (value === null || value === undefined) return undefined + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') { + if (dbType === 'ms_sql_server') return value ? '1' : '0' + return value ? 'TRUE' : 'FALSE' + } + if (typeof value !== 'string') return undefined + let escaped = value.replace(/'/g, "''") + // MySQL, Snowflake and BigQuery treat a backslash inside a string literal as + // an escape character. + if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') { + escaped = escaped.replace(/\\/g, '\\\\') + } + // A plain constant is varchar on SQL Server and goes through the database + // code page; the N prefix keeps it Unicode against nvarchar columns. + return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'` +} + +/** `"column" = ` predicate, or undefined when the value can't be + * rendered as a literal. */ +export function renderDbEqualityFilter( + column: string, + value: unknown, + dbType: DbType +): string | undefined { + const literal = renderDbLiteral(value, dbType) + if (literal === undefined) return undefined + return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}` +} + export function getLanguageByResourceType(name: string): ScriptLang { const language = { postgresql: 'postgresql', diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 8ef4ffbf95..565ac5d5d3 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -473,15 +473,6 @@ let appEditorHeader: AppEditorHeader | undefined = $state(undefined) - export function triggerTutorial() { - const urlParams = new URLSearchParams(window.location.search) - const tutorial = urlParams.get('tutorial') - - if (tutorial) { - appEditorHeader?.runTutorialById(tutorial) - } - } - let box: HTMLElement | undefined = $state(undefined) function parseScroll() { $yTop = box?.scrollTop ?? 0 diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index a57dc87622..ab043f81a1 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -7,25 +7,13 @@ import { redo, undo } from '$lib/history.svelte' import { discardDraftAfterDeploy } from '$lib/userDraftToast' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' - import { - enterpriseLicense, - tutorialsToDo, - userStore, - userWorkspaces, - workspaceStore - } from '$lib/stores' + import { enterpriseLicense, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { isMac, type Item, userPathPrefix } from '$lib/utils' - import { resetAllTodos, skipAllTodos } from '$lib/tutorialUtils' - import { getTutorialIndex } from '$lib/tutorials/config' import { random_adj } from '$lib/components/random_positive_adjetive' import { AlignHorizontalSpaceAround, BellOff, - BookOpen, Bug, - CheckCheck, - CheckCircle, - Circle, DiffIcon, Expand, FileJson, @@ -33,7 +21,6 @@ FormInput, History, Laptop2, - RefreshCw, Save, Smartphone, FileClock, @@ -61,7 +48,6 @@ import Awareness from '$lib/components/Awareness.svelte' import { secondaryMenuLeftStore, secondaryMenuRightStore } from './settingsPanel/secondaryMenu' import Dropdown from '$lib/components/DropdownV2.svelte' - import AppEditorTutorial from './AppEditorTutorial.svelte' import AppReportsDrawer from './AppReportsDrawer.svelte' import DebugPanel from './contextPanel/DebugPanel.svelte' @@ -679,49 +665,9 @@ action: () => { appExport?.open(toStatic($app, $staticExporter, $summary).app) } - }, - { - displayName: 'Tutorials', - icon: BookOpen, - separatorTop: true, - submenuItems: [ - { - displayName: 'Background runnables', - action: () => appEditorTutorial?.runTutorialById('backgroundrunnables'), - icon: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? Circle - : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? undefined - : 'green' - }, - { - displayName: 'Connection', - action: () => appEditorTutorial?.runTutorialById('connection'), - icon: $tutorialsToDo.includes(getTutorialIndex('connection')) ? Circle : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('connection')) ? undefined : 'green' - }, - { - displayName: 'Reset tutorials', - action: () => resetAllTodos(), - icon: RefreshCw, - separatorTop: true - }, - { - displayName: 'Skip tutorials', - action: () => skipAllTodos(), - icon: CheckCheck - } - ] } ]) as Item[] - let appEditorTutorial: AppEditorTutorial | undefined = $state(undefined) - - export function runTutorialById(id: string, options?: { skipStepsCount?: number }) { - appEditorTutorial?.runTutorialById(id, options) - } - let appReportingDrawerOpen = $state(false) export function openTroubleshootPanel() { @@ -1090,15 +1036,7 @@
    {/if}
    -
    - - {#if $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) || $tutorialsToDo.includes(getTutorialIndex('connection'))} - - {/if} -
    - +
    {#if hasErrors} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index f673711159..836af68db6 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -9,7 +9,10 @@ import ClipboardPanel from '$lib/components/details/ClipboardPanel.svelte' import { untrack } from 'svelte' - import { AppService, SettingService } from '$lib/gen' + import { AppService, SettingService, WorkspaceService } from '$lib/gen' + import type { GuestUsage } from '$lib/gen' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import Path from '$lib/components/Path.svelte' import { computeSecretUrl } from './appDeploy.svelte' import { base } from '$lib/base' @@ -22,6 +25,7 @@ } from '$lib/components/OnBehalfOfSelector.svelte' import { canUserBypassRuleKind, protectionRulesState } from '$lib/workspaceProtectionRules.svelte' import { FRONTEND_SDK_SCOPES } from '$lib/components/raw_apps/sdkScopes' + import { logFeatureUsage } from '$lib/utils/featureUsage' const WM_DEPLOYERS_GROUP = 'wm_deployers' @@ -90,6 +94,55 @@ (rulesetsLoaded && canUserBypassRuleKind('RestrictAnonymousAppDeployment', $userStore ?? undefined)) ) + let canSetGuest = $derived( + !!$userStore?.is_admin || + !!$userStore?.is_super_admin || + (rulesetsLoaded && + canUserBypassRuleKind('RestrictGuestAppDeployment', $userStore ?? undefined)) + ) + // The three rungs of the access control, widest last. `viewer` is a fourth + // execution mode that this control never sets (it runs components as the viewer, + // which a guest cannot be), so an app in it shows as members-only here. + let accessMode = $derived( + policy.execution_mode == 'anonymous' + ? 'anonymous' + : policy.execution_mode == 'guest' + ? 'guest' + : 'publisher' + ) + // Undefined until loaded. An app can be set to `guest` while the workspace has + // guests off, in which case the mode is stored but inert -- say so rather than + // letting the publisher believe the app is open. + let guestAccessEnabled: boolean | undefined = $state(undefined) + let guestUsage: GuestUsage | undefined = $state(undefined) + // Whether the deployment can have guests at all; off, the mode is not on offer. The + // backend decides; the hostname stands in until it has answered, the shared cloud + // being the only deployment where guests are unavailable. + let guestsAvailable = $derived.by(() => guestUsage?.available ?? !isCloudHosted()) + + $effect(() => { + const ws = opWs + if (ws === undefined) return + untrack(() => { + WorkspaceService.getPublicSettings({ workspace: ws }) + .then((s) => (guestAccessEnabled = s.guest_access_enabled)) + .catch(() => (guestAccessEnabled = undefined)) + WorkspaceService.getGuestUsage({ workspace: ws }) + .then((u) => (guestUsage = u)) + .catch(() => (guestUsage = undefined)) + }) + }) + + function onAccessModeChange(mode: string | undefined) { + if (mode === undefined || mode === accessMode) return + policy.execution_mode = mode + // Same as sandbox: a not-yet-deployed app has no row to PATCH, so + // `setPublishState` would 404. The mode is carried by the first deploy's + // policy; persist incrementally only once the app exists. + if (savedApp && !newApp) { + setPublishState() + } + } let canPreserve = $derived(!!$userStore?.is_admin || !!$userStore?.is_super_admin || isDeployer) let savedOnBehalfOfEmail = $derived(savedApp?.policy?.on_behalf_of_email) let savedOnBehalfOf = $derived(savedApp?.policy?.on_behalf_of) @@ -128,6 +181,10 @@ }${customPath}` ) + // The app URL a guest JWT rides on: append `guest.` and the viewer authenticates the + // token as a seatless guest. Uses the custom URL when set, else the public secret URL. + let guestJwtBase = $derived(customPath !== undefined ? fullCustomUrl : secretUrlHref) + // When embedding a raw app in an iframe inside another Windmill app (or any // cross-origin-isolated page), the embedded document must set COEP. The // `wm_coep` flag opts the public app into the cross-origin isolation headers. @@ -300,6 +357,12 @@ checked={policy.sandbox == true} on:change={(e) => { policy.sandbox = e.detail || undefined + // Counted where the toggle is flipped rather than where the policy is + // persisted: a not-yet-deployed app only mutates it locally, and skipping + // those would read as unused in the case where it is picked up front. + logFeatureUsage('app_sandbox', 'toggled', { + key: `${rawApp ? 'raw' : 'low_code'}:${e.detail ? 'on' : 'off'}` + }) // Frontend API access exists only for a sandboxed app, so turning // isolation off drops the declared scopes with it rather than leaving // them set but inert. @@ -389,34 +452,83 @@ {/if} {#if !hideSecretUrl} -

    Public URL

    +

    Access

    - {#if rulesetsLoaded && !canSetAnonymous} + {#if rulesetsLoaded && !canSetAnonymous && policy.execution_mode != 'anonymous'} - Making this app publicly accessible without login is restricted to workspace admins and - bypass users by a workspace protection rule + Opening this app to anyone with the link is restricted to workspace admins and bypass users + by a workspace protection rule + +
    + {/if} + {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest' && guestsAvailable} + + Opening this app to guests is restricted to workspace admins and bypass users by a workspace + protection rule
    {/if}
    - { - policy.execution_mode = e.detail ? 'anonymous' : 'publisher' - // Same as sandbox: a not-yet-deployed app has no row to PATCH, so - // `setPublishState` would 404. The mode is carried by the first - // deploy's policy; persist incrementally only once the app exists. - if (savedApp && !newApp) { - setPublishState() - } - }} - disabled={!savedApp || (!canSetAnonymous && policy.execution_mode != 'anonymous')} - /> + onAccessModeChange(e.detail)} + disabled={!savedApp} + > + {#snippet children({ item })} + + + + {/snippet} + +
    +
    + {#if policy.execution_mode == 'anonymous'} + Anyone holding the secret URL below can open this app without signing in. + {:else if policy.execution_mode == 'guest'} + {#if !guestsAvailable} + Guests are not available on Windmill Cloud, so this app still admits members only. They + require a self-hosted instance or a dedicated Windmill Cloud deployment. + {:else if guestUsage && !guestUsage.instance_enabled} + A superadmin has turned guests off for this instance, so this app still admits members + only. + {:else if guestAccessEnabled === undefined} + Checking whether this workspace allows guests… + {:else if guestAccessEnabled === false} + Guests are turned off for this workspace, so this app still admits members only. A + workspace admin can turn them on in the workspace settings. + {:else} + Anyone your identity provider authenticates can open this app without a Windmill account. + They join no workspace. Members of this workspace can open it too. + {#if guestUsage} + {guestUsage.guest_count} of {guestUsage.free_allowance} free guests used across this instance + in the last {guestUsage.window_days} days; beyond that, {guestUsage.metered + ? 'every four guests count as one seat' + : 'new guests are refused until the count drops'}. + {/if} + {/if} + {:else} + Only workspace members with read access on this app can open it. + {/if}
    {#if !savedApp || newApp} @@ -451,6 +563,38 @@ {/if}
    + {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase && guestsAvailable} +
    +
    + Embed for your own authenticated users (guest JWT) +
    +
    + To open this app for a user your own product already authenticates, mint a short-lived JWT + in your backend and append it to the app URL as guest.<jwt>. Each token + is its own seatless guest, confined to this app — no shared secret and no Windmill + account, unlike the plain secret URL above. +
    +
    + Windmill verifies the token against the workspace's guest JWT key (Workspace settings → + Guests) — a PEM public key or a JWKS URL{#if !isCloudHosted()}, or the instance's + configured issuer (JWT_EXT_JWKS_URL) when no workspace key is set{/if}. Set + the public half there; in your backend, sign each token with the matching + private key using RS256/384/512, PS256/384/512 or ES256/384 (symmetric HS* is + refused), carrying email, workspace_id = {opWs}, + app_path = {appPath} and exp (at most 24h ahead). +
    + +
    + Replace YOUR_GUEST_JWT with the token your backend signs per user. Past the instance's + free guest allowance a new guest email is refused (see the count above); guests already seen + in the window keep working. +
    +
    + {/if} +
    {#if !($userStore?.is_admin || $userStore?.is_super_admin)} diff --git a/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte b/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte deleted file mode 100644 index c268058717..0000000000 --- a/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte +++ /dev/null @@ -1,35 +0,0 @@ - - -) => { - targetTutorial = event.detail.detail - }} -/> - - { - targetTutorial = undefined - }} - on:confirmed={async () => { - window.open(`/apps/add?tutorial=${targetTutorial}`, '_blank') - }} -> -
    - This tutorial can only be run on a new app. -
    -
    diff --git a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte index 4ae67dbb0e..614d1f4fa4 100644 --- a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte +++ b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte @@ -9,10 +9,10 @@ cleanValueProperties, displayDate, emptyString, - orderedYamlStringify, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AppService, type AppWithLastVersion, type AppHistory } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { Skeleton } from '$lib/components/common' diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index d5839c6766..99ea07c8fd 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -22,19 +22,22 @@ let { workspace, - path, - editHref + path }: { workspace: string path: string - /** Where the Edit button points (low-code vs raw editor). */ - editHref: string } = $props() let app: any = $state(undefined) let notExists = $state(false) let noPermission = $state(false) let canWriteApp = $state(false) + /** Raw vs low-code, read from the app itself rather than from the route: + * both kinds render here and either route serves either kind (links to a raw + * app point at /apps/get all over the app), so only the app can say which + * editor the Edit button must open. */ + let isRawApp = $state(false) + let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined // The opaque iframe loads the dedicated cookieless, chrome-less viewer route. @@ -103,11 +106,14 @@ } } - // Edit button: determine write access on this real-origin page (cookie). + // Edit button: determine write access and which editor to open on this + // real-origin page (cookie). The sandboxed low-code app never loads on this + // page (it loads inside the opaque iframe), so `app` can't be the source. async function loadPerms() { try { const lite: any = await AppService.getAppLiteByPath({ workspace, path }) canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore) + isRawApp = !!lite?.raw_app } catch (_) { canWriteApp = false } diff --git a/frontend/src/lib/components/apps/editor/PublicApp.svelte b/frontend/src/lib/components/apps/editor/PublicApp.svelte index c0cf693e0f..6de8f9df0c 100644 --- a/frontend/src/lib/components/apps/editor/PublicApp.svelte +++ b/frontend/src/lib/components/apps/editor/PublicApp.svelte @@ -1,14 +1,14 @@ @@ -107,18 +111,6 @@ >Powered by   Windmill
    - - {#snippet userInfo(child)} -
    {child}
    - {/snippet} - -
    {#if $userStore} - {@render userInfo($userStore.username)} - {:else if globalUser} - {@render userInfo(globalUser.email)} - {:else}{/if} -
    {/if} {#if notExists} @@ -128,17 +120,31 @@
    {:else if noPermission} -
    This app requires read access
    -
    - {#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace} - You are logged in but are not a member of the workspace {effectiveWorkspace} this app is part of - {:else}You must be logged in and have read access to this app{/if}
    + {#if guestAppPath && !$userStore} +
    Sign in to open this app
    +
    + You do not need a Windmill account. Signing in lets you open this app and nothing else. +
    + {:else} +
    + This app requires read access +
    +
    + {#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace} + You are logged in but are not a member of the workspace {effectiveWorkspace} this app is part of + {:else}You must be logged in and have read access to this app{/if}
    + {/if}
    {#if !jwtError} - + {/if}
    {:else if app} @@ -170,27 +176,35 @@ )} style={app?.value?.['css']?.['app']?.['viewer']?.style} > - goto(path)} - gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} - /> + {#await loadAppPreview()} + + {:then Module} + goto(path)} + gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} + /> + {:catch} +
    + Reload the page to try again. +
    + {/await}
    {/if} {/key} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index 3ba1bac617..e994767488 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -23,15 +23,17 @@ * bearer token (no cookie); the raw wrapper document always carries `CSP: sandbox`. */ import { BROWSER } from 'esm-env' - import { OpenAPI } from '$lib/gen' + import { OpenAPI, UserService } from '$lib/gen' import { page } from '$app/state' import { onDestroy, onMount, setContext, type Snippet } from 'svelte' - import { Alert, Skeleton } from '$lib/components/common' + import Alert from '$lib/components/common/alert/Alert.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import { base } from '$app/paths' import { goto } from '$app/navigation' import Login from '$lib/components/Login.svelte' import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils' import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types' + import { loadAppPreview } from './loadAppPreview' import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte' import { hasStoredSdkConsent, storeSdkConsent } from '$lib/components/raw_apps/sdkScopes' @@ -49,7 +51,9 @@ fetchEmbedToken, onViewerReady, viewer, - viewerUrl + viewerUrl, + guestAppPath = undefined, + guestEntry = 'none' }: { /** Embedder-side: validate access + mint the scoped token. Throws with a * `.status` of 401 (login required) or 404 (not found). Pass @@ -68,6 +72,16 @@ * (`/apps/get`, auth-gated, with chrome) differs from the cookieless, * chrome-less viewer route (`/app_embed`). */ viewerUrl?: string + /** `/` when this app is open to guests. The embedder's + * login gate fires before the page's own load, so the page must resolve this + * up front and pass it down — otherwise a signed-out visitor is offered an + * ordinary sign-in that creates an account and still cannot open the app. */ + guestAppPath?: string | undefined + /** Whether the app admits guests, as far as the page has found out. The sign-in + * card waits while `pending` (a configured auto-login would otherwise start an + * ordinary sign-in) and refuses to offer one on `error` — a transient fault must + * not become an account and a seat. Nothing else waits on it. */ + guestEntry?: 'pending' | 'none' | 'guest' | 'error' } = $props() const EMBED_PARAM = 'wm_embed' @@ -194,7 +208,48 @@ } // ---------------------------- embedder mode ---------------------------- - let status: 'loading' | 'ready' | 'noPermission' | 'notExists' | 'sdkPrompt' = $state('loading') + type FrameStatus = 'loading' | 'ready' | 'noPermission' | 'notExists' | 'sdkPrompt' + let status = $state('loading') + /** Whether the visitor holds an account session, probed whenever the app denies + * them. An account this app still refuses is not something signing in again can + * fix — an identity with an account is never given a guest session — so the card + * gives way to an explanation. */ + let accountSession = $state<'unknown' | 'none' | 'held'>('unknown') + /** What the sign-in refused with, shown above the card until the next attempt. */ + let signInError: string | undefined = $state(undefined) + let deniedStatus: number | undefined = $state(undefined) + /** The sign-in card belongs on a 401, and on a 403 unless discovery has settled + * that the app is not open to guests: a 403 on a guest app is a session for another + * app of the workspace, which a fresh sign-in replaces. True while discovery is + * pending, so the skeleton shows rather than a flash of "Not found". */ + let offerSignIn = $derived( + status === 'noPermission' || + (status === 'notExists' && deniedStatus === 403 && guestEntry !== 'none') + ) + let signInDidNotHelp = $derived(offerSignIn && accountSession === 'held') + $effect(() => { + if (offerSignIn && accountSession === 'unknown') { + // Workspace-less, so it answers for an account and never for a guest + // (pinned to its workspace) or for nobody. + UserService.getCurrentEmail() + .then(() => (accountSession = 'held')) + .catch(() => (accountSession = 'none')) + } + }) + + // The stale guest session must be gone before the card mounts: it still + // authenticates here, so the popup's success poll would see it and complete the + // sign-in before the new session lands. The card waits on `staleGuestCleared`; a + // failed logout fails closed rather than offering a sign-in that cannot complete. + let staleGuestCleared = $state(false) + let staleGuestLogoutFailed = $state(false) + $effect(() => { + if (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared) { + UserService.logout() + .then(() => (staleGuestCleared = true)) + .catch(() => (staleGuestLogoutFailed = true)) + } + }) let embedToken: string | null = $state(null) let iframeEl: HTMLIFrameElement | undefined = $state(undefined) @@ -293,6 +348,10 @@ } finishReady() } catch (e: any) { + // 401: no session. 403 on an app that admits guests: a guest session for a + // different app of this workspace, which a fresh sign-in replaces. Either + // way the sign-in card is the answer; anything else is not found. + deniedStatus = e?.status status = e?.status === 401 ? 'noPermission' : 'notExists' } } @@ -363,7 +422,8 @@ if (unsandboxed || isRaw) { // Render the app directly on this origin: same-origin when unsandboxed // (the default), or a single opaque bundle iframe when it's a sandboxed - // raw app. + // raw app. A low-code app's runtime downloads alongside the app payload. + if (!isRaw) loadAppPreview().catch(() => {}) onViewerReady?.(undefined, requestTokenRefresh) } else { // Sandboxed low-code: hand the scoped token to the opaque viewer iframe. @@ -472,6 +532,8 @@ onMount(() => { if (isViewer) { + // Only a sandboxed low-code app is ever framed as a viewer. + loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() // Announce readiness so the embedder sends us the token. @@ -514,7 +576,7 @@ {/if} {:else if status === 'loading'} -{:else if status === 'notExists'} +{:else if status === 'notExists' && !offerSignIn}
    There was an error loading the app, is the url correct? @@ -529,17 +591,58 @@ onContinue={onSdkConsentContinue} onDecline={onSdkConsentDecline} /> -{:else if status === 'noPermission'} +{:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} + +{:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} +
    + + The app could not be reached to find out who may open it. Reload to try again. + +
    +{:else if offerSignIn} -
    This app requires read access
    -
    - initEmbedder()} - popup - rd={page.url.pathname + page.url.search + page.url.hash} - /> -
    + {#if signInDidNotHelp} + +
    + You are signed in, but this app is not open to you +
    +
    + It is open to the people it was shared with{guestAppPath + ? ', and to guests who have no Windmill account' + : ''}. Ask the person who shared it to give your account access. +
    + {:else} + {#if guestAppPath} +
    Sign in to open this app
    +
    + You do not need a Windmill account. Signing in lets you open this app and nothing else. +
    + {:else} +
    + This app requires read access +
    + {/if} + {#if signInError} +
    + {signInError} +
    + {/if} +
    + { + signInError = undefined + accountSession = 'unknown' + initEmbedder() + }} + onLoginError={(message) => (signInError = message)} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> +
    + {/if} {:else if unsandboxed} + {#if selectionDbtHasColumns} +
    + +
    + {/if} + + {#if selectionDbt.raw_code} +
    + +
    + {/if}
    {:else}
    diff --git a/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte new file mode 100644 index 0000000000..28afa5904e --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte @@ -0,0 +1,73 @@ + + + +{#snippet failure()} +
    + Part of this column lineage could not be loaded, so the trace may be incomplete. +
    +{/snippet} + +{#if loading && nodes.length === 0} +
    + + Loading column lineage +
    +{:else if nodes.length === 0} + {#if failed} +
    {@render failure()}
    + {/if} +{:else if graph} +
    + + {#if failed} + {@render failure()} + {/if} + {#if truncated} +
    + Showing the part of the trace nearest this relation. The lineage reaches further than one + view can draw. +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte new file mode 100644 index 0000000000..a1c769d9af --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte @@ -0,0 +1,50 @@ + + +{#if columns.length > 0} +
    +
    {analyzed ? 'columns' : 'columns declared'}
    +
    + {#each columns as col (col.name)} +
    + {col.name} + {#if col.type} + {col.type} + {/if} + {col.description} +
    + {/each} +
    + + {#if !analyzed} +
    + Declared metadata. Set `column_lineage: true` in the descriptor for the real column schema, + typed and in the order the model produces it. +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 3439fa5ad5..b71a51ea0a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -17,7 +17,9 @@ AssetGraphResponse, AssetGraphSelection, NativeTriggerKind, - PipelineMode, DbtAssetProvenance } from './types' + PipelineMode, + DbtAssetProvenance + } from './types' import type { AssetKind, Script, ScriptLang } from '$lib/gen' import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte' import type { PipelineOutputKind } from './pipelineTemplates' @@ -76,6 +78,9 @@ localScriptsVersion, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, + selectionColumnFailed = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -180,8 +185,16 @@ * the selected node's source on live-reload. */ localScriptsVersion?: unknown selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> - /** Transitive column-lineage trace for a selected ducklake asset (route page). */ + /** Transitive column-lineage trace for the selected asset (route page). */ selectionColumnGraph?: ColumnLineageGraph + /** That trace still being fetched — a dbt relation's is a request of its + * own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** That trace cut at the part nearest the selection. */ + selectionColumnTruncated?: boolean + /** That trace could not be fetched. Distinguished from an empty one: a + * project without the analysis pass draws nothing either. */ + selectionColumnFailed?: boolean /** dbt provenance of the selected relation — carries its SQL. */ selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean @@ -514,6 +527,9 @@ selection={activeDraft ? undefined : editor.selection} selectionProducers={activeDraft ? [] : selectionProducers} {selectionColumnGraph} + {selectionColumnLoading} + {selectionColumnTruncated} + {selectionColumnFailed} {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts index 6842dcfd86..16c8713c5c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import type { AssetGraphResponse } from './types' import { buildColumnGraph, + buildDbtColumnGraph, colNodeId, + mergeColumnGraphs, + type ColumnLineageGraph, traceColumn, connectedComponent, assetColumnNodes, @@ -120,6 +123,79 @@ describe('buildColumnGraph', () => { }) }) +describe('buildDbtColumnGraph', () => { + it('takes the direct kinds and drops any other', () => { + // `scan` means the column was read to produce the ROW — a join key, a + // predicate, a `group by` — so it reaches every output column of its model + // and is not what a column trace means. The server filters it out; this + // filters again, because the kind set is the engine's and an unknown one + // must not become an edge the trace calls data flow. + const g = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw_name', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'mod' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'id', + kind: 'copy' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'scan' + } + ]) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'clean_name'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw_name')]) + ) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'id'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'id')]) + ) + }) +}) + +describe('mergeColumnGraphs', () => { + it('chains a dbt column into what a producer derives from it', () => { + // The two halves arrive separately — the producer's from the asset graph, + // dbt's from its own request — and meet at the dbt node a `// column` + // annotation names. A trace has to cross that, or a dbt selection stops + // before the script consuming it. + const dbt = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw', + to_asset_path: 'main/s/mart', + to_column: 'clean', + kind: 'copy' + } + ]) + const producer: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() + } + const src = colNodeId('dbt', 'main/s/mart', 'clean') + const out = colNodeId('ducklake', 'wh/report', 'total') + producer.nodes.set(src, { kind: 'dbt', path: 'main/s/mart', column: 'clean' }) + producer.nodes.set(out, { kind: 'ducklake', path: 'wh/report', column: 'total' }) + producer.up.set(out, new Set([src])) + producer.down.set(src, new Set([out])) + + const merged = mergeColumnGraphs(dbt, producer) + expect(traceColumn(colNodeId('dbt', 'main/s/stg', 'raw'), merged)).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw'), src, out]) + ) + }) +}) + describe('traceColumn', () => { it('returns the full upstream + downstream impact set of a source column', () => { const g = buildColumnGraph(chainGraph()) diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts index 50fa1c4e7b..54837753e1 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts @@ -1,6 +1,11 @@ -import type { AssetKind } from '$lib/gen' +import type { AssetKind, DbtColumnLineage } from '$lib/gen' import type { AssetGraphResponse } from './types' +// One column-to-column edge of a dbt project's static analysis, as the API +// serves it. Taken from the generated client rather than restated: unlike the +// asset graph, this response is fetched through it. +export type DbtColumnEdge = DbtColumnLineage['edges'][number] + // A node in the column-level lineage graph: one column of one asset. export type ColumnNode = { kind: AssetKind; path: string; column: string } export type ColumnNodeId = string @@ -23,6 +28,21 @@ export type ColumnLineageGraph = { down: Map> } +export const EMPTY_COLUMN_GRAPH: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() +} + +// Direct value flow, as dbt's static analysis labels it: `copy` passes a column +// through, `mod` transforms it. The API serves only those two — the third kind, +// `scan`, means the column was read to produce the ROW rather than the value (a +// join key, a `where` predicate, a `group by`), so it reaches EVERY output +// column of the model and would draw the diagram as a complete bipartite graph. +// Filtered here as well so a kind the engine invents cannot silently become an +// edge the trace claims is data flow. +const DIRECT_DBT_LINEAGE = new Set(['copy', 'mod']) + // Build the column graph from a resolved asset graph. A producer's // `column_lineage` describes the columns of the asset it materializes; that // output asset is the ducklake target it writes (v1 materialize target), found @@ -89,6 +109,59 @@ export function buildColumnGraph(graph: AssetGraphResponse): ColumnLineageGraph return { nodes, up, down } } +// The same graph, from dbt's own column lineage. dbt arrives already resolved to +// two relations rather than anchored to a producer, and the API serves only the +// direct kinds, so this is a straight edge list. +export function buildDbtColumnGraph(edges: DbtColumnEdge[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + const addNode = (n: ColumnNode): ColumnNodeId => { + const id = colNodeId(n.kind, n.path, n.column) + if (!nodes.has(id)) nodes.set(id, n) + return id + } + for (const e of edges) { + // Belt and braces: the API filters to `copy`/`mod`, and a kind an engine + // invents must not silently become an edge the trace calls data flow. + if (!DIRECT_DBT_LINEAGE.has(e.kind)) continue + const src = addNode({ kind: 'dbt', path: e.from_asset_path, column: e.from_column }) + const out = addNode({ kind: 'dbt', path: e.to_asset_path, column: e.to_column }) + if (src === out) continue + ;(up.get(out) ?? up.set(out, new Set()).get(out)!).add(src) + ;(down.get(src) ?? down.set(src, new Set()).get(src)!).add(out) + } + return { nodes, up, down } +} + +// One graph out of several, so a trace crosses the boundary between them. +// +// The two halves reach each other through shared node ids: a producer's +// `// column out <- dbt://wh/schema/model.col` puts a `('dbt', path, column)` +// node in the producer graph under the same `colNodeId` the dbt lineage mints +// for it, so the union chains a dbt model's columns into the script that +// consumes them and on into what that script writes. Kept separate up to here +// because they are fetched separately — the producer half rides on the asset +// graph, the dbt half is asked for per selection. +export function mergeColumnGraphs(...graphs: ColumnLineageGraph[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + for (const g of graphs) { + for (const [id, n] of g.nodes) if (!nodes.has(id)) nodes.set(id, n) + for (const [dir, into] of [ + [g.up, up], + [g.down, down] + ] as const) { + for (const [id, adj] of dir) { + const target = into.get(id) ?? into.set(id, new Set()).get(id)! + for (const m of adj) target.add(m) + } + } + } + return { nodes, up, down } +} + // Every node reachable from `start` by following `adj` (transitive closure, // excluding `start` itself). Iterative to avoid deep-recursion limits. function reach(start: ColumnNodeId, adj: Map>): Set { diff --git a/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts new file mode 100644 index 0000000000..477ed630b1 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts @@ -0,0 +1,153 @@ +import { AssetService, JobService, type DbtColumnLineage } from '$lib/gen' +import { + buildDbtColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph +} from './columnLineageGraph' + +/** Which stored dbt graph a view is drawing. A job — a run's snapshot, or the + * editor's parse of its own buffer — is asked through the job route, the only + * way to reach a graph that names no deployed version; otherwise the deployed + * version by hash, or the current one when there is no hash. */ +export type DbtGraphPin = { jobId?: string; scriptHash?: string | number } + +/** What a selection's dbt column lineage is doing right now. `loading` is + * separate because a project still being fetched and one that never asked for + * the analysis pass are the same empty graph otherwise. */ +export type DbtColumnLineageState = { + readonly graph: ColumnLineageGraph + readonly loading: boolean + /** The component reaches past what `graph` holds — the API cut it at the + * part nearest the selection. */ + readonly truncated: boolean + /** The request failed, so `graph` is empty for a reason that is not "this + * project has no column lineage". */ + readonly failed: boolean +} + +function fetchLineage( + workspace: string, + assetPaths: string[], + pin: DbtGraphPin | undefined +): Promise { + return pin?.jobId + ? JobService.getDbtRunColumnLineage({ workspace, id: pin.jobId, assetPath: assetPaths }) + : AssetService.getDbtColumnLineage({ + workspace, + assetPath: assetPaths, + dbtScriptHash: pin?.scriptHash != undefined ? String(pin.scriptHash) : undefined + }) +} + +/** Follow the selection, fetching the dbt column lineage it reaches. + * + * One request per selection, whatever it reaches: the API takes every relation + * at once and walks out from all of them, so there is no partial answer to hold + * on to between selections and nothing to go stale behind a redeploy. + * + * Per selection rather than off the graph response: the graph is folder-wide + * and a run page polls it, while this is drawn for one selection. It also means + * the request is never made for a project that did not opt into the analysis + * pass — the pane simply never shows the section. + */ +export function useDbtColumnLineage(args: { + workspace: () => string | undefined + /** The dbt relations to expand. The selection itself when it is one; for a + * selection of another kind, every dbt relation its own lineage reaches — + * a ducklake table can be derived from several, and expanding only the + * first would leave the rest as leaves. */ + assetPaths: () => string[] + /** The graph on screen, so the lineage describes the same project. */ + pin?: () => DbtGraphPin | undefined + /** Which fetch of that graph is on screen. It changes when the view goes and + * gets the graph again — a Refresh, a deploy — and asking again is the whole + * point: the relation, the pin and the seeds are all unchanged by a + * redeploy, so without this the pane would pair a freshly fetched model's + * SQL and columns with the edges of the version before it. It is also what + * retries a request that failed. */ + generation?: () => unknown +}): DbtColumnLineageState { + let graph = $state(EMPTY_COLUMN_GRAPH) + let loading = $state(false) + let truncated = $state(false) + let failed = $state(false) + + // The question the state in hand answers, and a counter deciding which answer + // is still wanted. Neither is a cache of edges: the API returns a whole + // component, so an answer is either the current selection's or nothing. + let asked: string | undefined = undefined + let latest = 0 + + $effect(() => { + const workspace = args.workspace() + const paths = [...new Set(args.assetPaths())].sort() + const pin = args.pin?.() + const question = JSON.stringify([ + workspace, + pin?.jobId, + pin?.scriptHash, + args.generation?.() ?? null, + paths + ]) + // A selection re-derived from a graph that polled is the same question. Not + // asking it again is what keeps a run page from refetching a component's + // worth of edges every poll to redraw what is already on screen — while a + // graph the view deliberately went and fetched moves `generation`, so that + // IS a new question. + if (question === asked) return + asked = question + const id = ++latest + if (!workspace || paths.length === 0) { + graph = EMPTY_COLUMN_GRAPH + truncated = false + failed = false + loading = false + return + } + // The previous question's failure is not this one's. Cleared as the request + // goes out rather than when it lands, or a retry keeps saying the trace is + // incomplete while it is being fetched, and a new selection inherits the + // last one's failure until its own answer arrives. + failed = false + loading = true + fetchLineage(workspace, paths, pin).then( + (r) => { + if (id !== latest) return + graph = buildDbtColumnGraph(r?.edges ?? []) + truncated = r?.truncated ?? false + failed = false + loading = false + }, + // Lineage annotates a graph that renders without it, so a failed fetch + // leaves that branch unexpanded rather than putting an error over the + // model — but it SAYS so, because an empty trace is what a project + // without the analysis pass looks like, and the two must not read + // alike. Not retried on its own: the effect reruns whenever the canvas + // redraws, and a failing endpoint would then be asked once per redraw. + // Selecting another node and back asks again, and where a caller + // passes `generation`, so does going and fetching the graph. + () => { + if (id !== latest) return + graph = EMPTY_COLUMN_GRAPH + truncated = false + failed = true + loading = false + } + ) + }) + + return { + get graph() { + return graph + }, + get loading() { + return loading + }, + get truncated() { + return truncated + }, + get failed() { + return failed + } + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index de00aef2be..c57687d78b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -319,6 +319,18 @@ describe('resolveGraph', () => { expect(assetTrigKeys(r, 'f/x/open')).toEqual(['ducklake:main.orders']) }) + // A relation a native `// materialize manual dbt://…` script writes wakes its + // subscribers, so hiding the edge would leave the author's own annotation off + // the canvas. The deploy refuses the ones that cannot fire. + it('draws an explicit dbt:// subscription as an unsaved trigger overlay', () => { + const liveAnnotations = { + scriptPath: 'f/x/open', + annotations: ann({ triggerAssets: [{ kind: 'dbt', path: 'main/analytics/orders' }] }) + } + const r = resolveGraph(input({ liveAnnotations })) + expect(assetTrigKeys(r, 'f/x/open')).toEqual(['dbt:main/analytics/orders']) + }) + it('open-script live annotations add unsaved triggers, deduped vs persisted', () => { const base = baseGraph({ triggers: [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index 867fa6f036..5c8aa697a4 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -77,23 +77,15 @@ function persistedNativeKinds(base: AssetGraphResponse, path: string): Set = new Set(['ducklake', 's3object']) -/** Whether a subscription on this kind can ever fire once deployed. - * - * A `dbt://` one cannot: dbt is the only producer of a warehouse relation and - * a dbt run does not dispatch, so the deploy refuses `// on dbt://…` outright - * (`scripts.rs`). The editor must not draw an arrow the deploy will reject — - * applied to the EXPLICIT overlays; auto-derivation is already scoped by - * `AUTO_TRIGGER_KINDS`. Parsing is left alone so the Rust-parity test still - * compares like for like. */ -function canTrigger(kind: AssetKind): boolean { - return kind !== 'dbt' -} - /** `kind:path` refs of a script's `// materialize` write target(s) (base + * the scd2 `_current` companion), which the body `SELECT` doesn't express. */ function materializeWriteRefs(parsed: PipelineAnnotations): string[] { @@ -381,7 +373,7 @@ function makeContext(input: ResolveGraphInput): ResolveContext { const liveRefKeys = new Set() if (openIsSavedEdit) { if (liveAnnotations.scriptPath === openPath) { - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) + for (const a of liveAnnotations.annotations.triggerAssets) liveRefKeys.add(`${a.kind}:${a.path}`) // The `// materialize ` target is a declared *output*, but it // lives in an annotation (not the SQL body), so neither triggerAssets @@ -625,7 +617,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // stable when the user clicks off this draft. Live annotations // (below) take over for the currently-open draft so keystroke // edits still update in real time. - for (const a of parsed.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of parsed.triggerAssets) { extraTriggers.push({ trigger_kind: 'asset', asset_kind: a.kind, @@ -704,7 +696,7 @@ function applyLiveBufferOverlay(acc: Accumulator, input: ResolveGraphInput, ctx: for (let i = extraTriggers.length - 1; i >= 0; i--) { if (extraTriggers[i].runnable_path === livePath) extraTriggers.splice(i, 1) } - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of liveAnnotations.annotations.triggerAssets) { const key = `${a.kind}:${a.path}` if (assetKeys.has(key)) continue extraTriggers.push({ diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 4ada1feec3..023f71e218 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -36,9 +36,14 @@ export interface DbtAssetProvenance { tags?: string[] description?: string data_tests?: DbtDataTest[] - /** Declared column metadata (name -> description). NOT column lineage: - * `manifest.json` carries none (docs/dbt-runtime.md, decision 14). */ + /** Declared column metadata (name -> description): what `manifest.json` + * carries, which is only the columns an author wrote down. */ columns?: Record + /** Every column of the relation, typed and in the order the model produces + * them, from the engine's static analysis. Present only for a project that + * opted into it (`column_lineage: true`); `manifest.json` has no such + * thing. Lockstep with Rust `DbtAssetProvenance.column_schema`. */ + column_schema?: { name: string; type?: string }[] /** A source's declared freshness policy. */ freshness?: unknown /** The model's SQL as written — the transform behind the node. Read-only: diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 23531d71f2..93367a725e 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -96,10 +96,11 @@ export function formatAssetKind(asset: { case 'volume': return 'Volume' case 'dbt': - // The SCHEME says dbt because dbt is the only thing that creates one; - // the PATH stays the relation, so a mart one project builds and the - // `source` the next reads land on one node — their dbt `unique_id`s - // differ where the relation does not (docs/dbt-runtime.md, decision 11). + // The SCHEME says dbt because dbt is what derives these relations; the + // PATH stays the relation, so a mart one project builds, the `source` + // the next reads, and a native `// materialize manual dbt://…` writer + // land on one node — their dbt `unique_id`s differ where the relation + // does not (docs/dbt-runtime.md, decision 11). return 'dbt table' } } diff --git a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte index 213beda001..96d341dc72 100644 --- a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte +++ b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte @@ -216,6 +216,8 @@ INSTANCE_GROUPS_SCIM_CREATE: 'instance_groups.scim_create', INSTANCE_GROUPS_SCIM_DELETE: 'instance_groups.scim_delete', INSTANCE_GROUPS_SCIM_UPDATE: 'instance_groups.scim_update', + INSTANCE_GROUPS_JIT_ADDUSER: 'instance_groups.jit_adduser', + INSTANCE_GROUPS_JIT_REMOVEUSER: 'instance_groups.jit_removeuser', VARIABLES_DECRYPT_SECRET: 'variables.decrypt_secret', WORKSPACES_READ_ENCRYPTION_KEY: 'workspaces.read_encryption_key', WORKSPACES_EDIT_COMMAND_SCRIPT: 'workspaces.edit_command_script', diff --git a/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte new file mode 100644 index 0000000000..dadf814a80 --- /dev/null +++ b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte @@ -0,0 +1,323 @@ + + + dispatch('canceled')} + on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })} +> +
    + {#if draftTriggers.length > 0} +
    +
    + {`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`} +
    + +
    5 ? 'h-[300px]' : ''}> + +
    + + + + + + + {#each draftTriggers as trigger} + {@const SvelteComponent = triggerIconMap[trigger.type]} + {@const permission = checkSavePermissions(trigger)} + {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} + + + + + + {/each} + + + + + {/if} + + {#if draftAgents.length > 0} +
    +
    + Saved agents this flow uses have unsaved changes. Select which ones to deploy with the + flow. An agent kept as a draft stays editable, and the flow runs the agent as currently + deployed. +
    + +
    5 ? 'h-[300px]' : ''}> + +
    + + + + + + + + {#each draftAgents as agent (agent.path)} + {@const permission = checkAgentPermissions(agent)} + {@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)} + + + + + + + + {/each} + + + + + {/if} + + diff --git a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte deleted file mode 100644 index cfebae0e9f..0000000000 --- a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte +++ /dev/null @@ -1,168 +0,0 @@ - - - dispatch('canceled')} - on:confirmed={() => dispatch('confirmed', { selectedTriggers })} -> -
    -
    - {`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed - draft triggers will be permanently deleted.`} -
    - -
    5 ? 'h-[300px]' : ''}> - -
    - - - - - - - {#each draftTriggers as trigger} - {@const SvelteComponent = triggerIconMap[trigger.type]} - {@const permission = checkSavePermissions(trigger)} - {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} - - - - - - {/each} - - {#if draftTriggers.length === 0} - - - - {/if} - - - - - diff --git a/frontend/src/lib/components/common/fileInput/FileInput.svelte b/frontend/src/lib/components/common/fileInput/FileInput.svelte index 11f7a0caa9..bde73b8b69 100644 --- a/frontend/src/lib/components/common/fileInput/FileInput.svelte +++ b/frontend/src/lib/components/common/fileInput/FileInput.svelte @@ -201,6 +201,13 @@ } } + /** Open the file chooser without the dropzone being clicked, for a caller whose + * affordance is a button elsewhere. The component is still what reads and filters + * the files, so the two paths cannot drift. */ + export function openPicker() { + input?.click() + } + export function clearFiles() { files = undefined dispatchChange() diff --git a/frontend/src/lib/components/common/index.ts b/frontend/src/lib/components/common/index.ts index 531036c3de..bb0268b39b 100644 --- a/frontend/src/lib/components/common/index.ts +++ b/frontend/src/lib/components/common/index.ts @@ -20,6 +20,7 @@ export { default as TabFade } from './tabs/TabFade.svelte' export { default as Tabs } from './tabs/Tabs.svelte' export { default as Breadcrumb } from './breadcrumb/Breadcrumb.svelte' export { default as FileInput } from './fileInput/FileInput.svelte' +export { default as ListRow } from './listRow/ListRow.svelte' export { default as RadioCard } from './radioCard/RadioCard.svelte' export { default as Section } from '../Section.svelte' export { default as Url } from './Url.svelte' diff --git a/frontend/src/lib/components/common/listRow/ListRow.svelte b/frontend/src/lib/components/common/listRow/ListRow.svelte new file mode 100644 index 0000000000..dfa56d5af3 --- /dev/null +++ b/frontend/src/lib/components/common/listRow/ListRow.svelte @@ -0,0 +1,142 @@ + + + +{#snippet body()} +
    + {#if icon} +
    {@render icon()}
    + {/if} +
    +
    + {@render title()} +
    + {#if subtitle} + + + {@render subtitle()} + + {/if} +
    +
    +{/snippet} + +{#if trailing} + +
    + {#if onClick} + + + {:else} + +
    {@render body()}
    + {/if} + {@render trailing()} +
    +{:else if !onClick} + + +
    + {@render body()} +
    +{:else} + +{/if} diff --git a/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts new file mode 100644 index 0000000000..c7325698e3 --- /dev/null +++ b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts @@ -0,0 +1,97 @@ +import { untrack } from 'svelte' + +/** + * The highlighted row of a searchable list: the one the arrow keys move and Enter + * activates, rendered by passing `highlighted` to `ListRow`. + * + * Pairs with a search field above the list — the arrows and Enter are answered while + * focus stays in it, so a query and a choice are one uninterrupted sequence. + */ +export function useListHighlight(opts: { + /** How many rows the list holds right now. */ + count: () => number + /** The DOM id of the row at this index — the same `id` given to its `ListRow`. */ + rowId: (index: number) => string + /** Where the highlight belongs when the list changes underneath it: the top hit while + * a search is on, and typically -1 (nothing lit) when it is not. */ + restingIndex: () => number + /** Open the row at this index. */ + onActivate: (index: number) => void + /** Ids of the elements whose Enter also activates the highlighted row — the search + * field. A focused row activates itself, so it is not one of these. */ + activateEnterFrom?: string[] +}) { + let index = $state(-1) + // Scrolling rows under a resting pointer makes the browser fire `mouseenter` on each + // one, which would drag the highlight back under the cursor as the arrow keys move it. + // Only a real pointer move hands the highlight back to the mouse. + let pointerOwns = $state(true) + + // Filtering reshuffles the rows under the highlight, so it goes back where the caller + // says it belongs rather than staying on a position that now means another row. + $effect(() => { + opts.count() + const resting = opts.restingIndex() + untrack(() => (index = resting)) + }) + + function land(next: number) { + const count = opts.count() + if (count === 0 || next < 0 || next >= count) return + pointerOwns = false + // Rows are tabbable, so focus can sit on one. Enter then activates whatever is + // focused, which has to stay the highlighted row — so any row counts, not just + // the lit one. Tab from the search field lands on the first row while the + // highlight rests on the best match, and testing only the lit row would leave + // focus behind and activate the wrong one. + const focusedId = document.activeElement?.id + const rowWasFocused = + !!focusedId && Array.from({ length: count }, (_, i) => opts.rowId(i)).includes(focusedId) + index = next + const row = document.getElementById(opts.rowId(index)) + row?.scrollIntoView({ block: 'nearest' }) + if (rowWasFocused) row?.focus() + } + + function move(delta: number) { + const count = opts.count() + if (count === 0) return + land(index < 0 ? (delta > 0 ? 0 : count - 1) : (index + delta + count) % count) + } + + return { + get index() { + return index + }, + /** Step the highlight, for a list whose own keys move it beyond Up and Down — + * a tree stepping into the children a folder just revealed. */ + move, + /** Put the highlight on a row named outright, rather than a step from wherever + * it is — the row a caller's own key landed on, or the one that has focus. A + * step cannot say this: from nothing lit it can only reach an end of the list. */ + moveTo: land, + /** Wire to each row's `onMouseEnter`. */ + hovered(i: number) { + if (pointerOwns) index = i + }, + /** Wire to the list container's `onpointermove`. */ + pointerMoved() { + pointerOwns = true + }, + /** Wire to the container that holds the search field and the rows, so the keys are + * answered whichever of the two has focus. */ + onKeydown(e: KeyboardEvent) { + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { + e.preventDefault() + move(e.key === 'ArrowDown' ? 1 : -1) + } else if ( + e.key === 'Enter' && + opts.activateEnterFrom?.includes((e.target as HTMLElement)?.id) && + index >= 0 + ) { + e.preventDefault() + opts.onActivate(index) + } + } + } +} diff --git a/frontend/src/lib/components/common/modal/Modal.svelte b/frontend/src/lib/components/common/modal/Modal.svelte index 6755dfb3fc..bcc4f33607 100644 --- a/frontend/src/lib/components/common/modal/Modal.svelte +++ b/frontend/src/lib/components/common/modal/Modal.svelte @@ -35,9 +35,11 @@ style?: string cancelText?: string | undefined kind?: 'button' | 'X' - /** Where you are inside the dialog, as a breadcrumb replacing the title: the whole path, - * the dialog's own root first. A dialog at its root passes nothing (or one level) and keeps - * its plain title; ancestors with an `onclick` are the way back, which Escape also takes. */ + /** Where you are inside the dialog: the whole path, the dialog's own root first. Below the + * root the last level becomes the heading and the ones above it the line under it, with a + * back control for the nearest. A dialog at its root passes nothing (or one level) and + * keeps its plain title; ancestors with an `onclick` are the way back, which Escape also + * takes. */ trail?: ModalTrailSegment[] /** A line under the title saying what the dialog is for; in the header so it does not * scroll away with the body. */ @@ -63,9 +65,12 @@ * side panel when it is open. Pass an explicit value to stack above other * surfaces (e.g. a modal opened over the /sessions preview-pane editor). */ minZIndex?: number - /** Rendered against the dialog's own name, before any level below it: what it marks is the - * dialog rather than wherever in it you have navigated to. */ + /** Rendered against the dialog's own name, wherever that name is: the heading at the root, + * the first level of the way back below it. */ titleBadge?: import('svelte').Snippet + /** Rendered against the level you are on, which below the root is the heading. Nothing at + * the root, where that level is the dialog and `titleBadge` already names it. */ + levelBadge?: import('svelte').Snippet settings?: import('svelte').Snippet children?: import('svelte').Snippet actions?: import('svelte').Snippet @@ -85,6 +90,7 @@ enterConfirms = true, minZIndex: minZIndexProp = undefined, titleBadge, + levelBadge, settings, children: children_render, actions @@ -101,7 +107,9 @@ // A trail of one level is the dialog at its root, which the plain title already shows. const crumbs = $derived(trail && trail.length > 1 ? trail : undefined) - // The level under the one you are on: what Escape and the back chevron return to. + // The level you are on, which is the heading below the root. + const current = $derived(crumbs?.[crumbs.length - 1]) + // The level under it: what Escape and the back control return to. const back = $derived(crumbs?.[crumbs.length - 2]) const dispatch = createEventDispatcher() @@ -164,22 +172,19 @@ - -{#snippet crumb(segment: ModalTrailSegment, isBack: boolean)} - + {label} + {@render badge?.()} + {/snippet} @@ -234,72 +239,69 @@
    - {#if crumbs} - -
    - - - {@render settings?.()} -
    - {:else} -
    -

    - {title} - {@render titleBadge?.()} -

    - {@render settings?.()} -
    - {/if} + {#if segment.onclick} + + {:else} + {segment.label} + {/if} + {#if i === 0} + {@render titleBadge?.()} + {/if} + {/each} + +
    + + {:else} + {@render heading(title, titleBadge, false)} + {/if} + {@render settings?.()} + {#if description}

    {description}

    diff --git a/frontend/src/lib/components/common/modal/Modal2.svelte b/frontend/src/lib/components/common/modal/Modal2.svelte index f33b1dec04..b9e57740b9 100644 --- a/frontend/src/lib/components/common/modal/Modal2.svelte +++ b/frontend/src/lib/components/common/modal/Modal2.svelte @@ -26,6 +26,10 @@ * and clicks "outside" the child would otherwise propagate * here and close the underlying modal. */ closeOnOutsideClick?: boolean + /** Close on Escape. Default true. Every open modal listens on the + * window, so a stacked pair would both close on one press; set it + * false on the underlying modal while its child is up. */ + closeOnEscape?: boolean /** Wider side padding and a lighter title, for a dialog whose body is a form rather * than a list. Opt-in: every other Modal2 keeps the padding and heading it had. */ formStyling?: boolean @@ -46,6 +50,7 @@ fixedHeight = 'md', contentClasses = '', closeOnOutsideClick = true, + closeOnEscape = true, formStyling = false, headerLeft, headerRight, @@ -80,7 +85,7 @@ } function handleKeyDown(event: KeyboardEvent) { - if (!isOpen) return + if (!isOpen || !closeOnEscape) return if (event.key === 'Escape') { event.preventDefault() event.stopPropagation() diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte index 430a53e66f..bfae0473f1 100644 --- a/frontend/src/lib/components/common/modal/PagedContent.svelte +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -4,7 +4,18 @@ /** One level of a paginated dialog. Order is the order given: the page on screen sits at rest * and every other waits off the side it is listed on, so a deeper page arrives from the right * and the way back arrives from the left without anyone naming a direction. */ - export type ModalPage = { key: string; content: Snippet } + export type ModalPage = { + key: string + content: Snippet + /** + * Drawn in place of `content` for a page that has not been opened yet, so the first + * navigation to it has something to slide in — without one, the box arrives empty and + * fills a frame later, which reads as the animation being broken rather than as + * loading. A skeleton is enough: it is on screen for the length of the transition. + * Unnecessary under `warm`, which builds every page up front. + */ + placeholder?: Snippet + } -
    + +
    {#each groups as group (group.title)}
    {#if group.title} diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 82a81e2eb0..938086a0e8 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -9,7 +9,14 @@ import type ShareModal from '$lib/components/ShareModal.svelte' import { ScriptService, type Script } from '$lib/gen' - import { userStore, userWorkspaces, workspaceStore } from '$lib/stores' + import { + disableHubStore, + hubBaseUrlStore, + userStore, + userWorkspaces, + workspaceStore + } from '$lib/stores' + import { scriptToHubUrl } from '$lib/hub' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { createEventDispatcher } from 'svelte' @@ -32,6 +39,7 @@ FolderOpen, ChevronUpSquare, GitFork, + Globe2, List, Pen, Shield, @@ -47,7 +55,12 @@ import Popover from '$lib/components/Popover.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' - import { editInForkAllowed, editInForkLabel, onEditInForkClick } from '$lib/utils/editInFork' + import { + claimTab, + editInForkAllowed, + editInForkLabel, + onEditInForkClick + } from '$lib/utils/editInFork' import EditInForkButton from './EditInForkButton.svelte' import { isCloudHosted } from '$lib/cloud' @@ -418,6 +431,32 @@ copyToClipboard(script.path) } }, + { + displayName: 'Publish to Hub', + icon: Globe2, + action: async () => { + // The row only carries metadata, so the code has to be fetched first; the tab is + // claimed before that, since Safari won't open one after an await. + const tab = claimTab() + try { + const fullScript = await ScriptService.getScriptByPath({ + workspace: $workspaceStore!, + path: script.path + }) + const url = scriptToHubUrl(fullScript, $hubBaseUrlStore).toString() + if (tab) { + tab.show(url) + } else if (!window.open(url)) { + sendUserToast('Allow popups to publish this script to the Hub', true) + } + } catch (e: any) { + tab?.discard() + sendUserToast(`Could not load ${script.path}: ${e?.body ?? e?.message ?? e}`, true) + } + }, + // Operators can't write scripts, so they have nothing to publish. + hide: $disableHubStore || $userStore?.operator + }, { displayName: script.archived ? 'Unarchive' : 'Archive', icon: Archive, diff --git a/frontend/src/lib/components/common/tabs/Tab.svelte b/frontend/src/lib/components/common/tabs/Tab.svelte index a652525b45..edb1135bbc 100644 --- a/frontend/src/lib/components/common/tabs/Tab.svelte +++ b/frontend/src/lib/components/common/tabs/Tab.svelte @@ -99,6 +99,7 @@ }} {disabled} {id} + data-tab-selected={isSelected ? 'true' : undefined} >
    ('[data-tab-selected="true"]') + if (el) lastX = el.offsetLeft + bar = { x: lastX, w: el ? el.offsetWidth : 0 } + } + + // Placing the bar for the first paint. Every later move comes from the observers below: a + // Tab marks itself selected in its own update, which has not run when an effect here does, + // so measuring from this side alone lands the bar on the tab that was selected before. + $effect(() => { + if (!slidingIndicator) return + void row + measureBar() + }) + + $effect(() => { + if (!slidingIndicator || !row) return + const ro = new ResizeObserver(measureBar) + ro.observe(row) + // The mark moving is the selection changing, and a tab added or removed changes what the + // bar has to sit on. Text counts too: a label rewritten in place — a count arriving, + // Result becoming Error — resizes the tab under the bar without touching the tree, and + // Svelte writes it straight to the node, so childList never sees it. + const mo = new MutationObserver(measureBar) + mo.observe(row, { + childList: true, + subtree: true, + characterData: true, + attributes: true, + attributeFilter: ['data-tab-selected'] + }) + return () => { + ro.disconnect() + mo.disconnect() + } + }) + let hashValues = $derived(values ? values.map((x) => '#' + x) : undefined) function hashChange() { @@ -77,10 +135,26 @@ {#if !hideTabs} - -
    +
    {@render children?.({ selected })} + {#if slidingIndicator} + + {/if}
    {/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index f610dd5ff4..7de7dd32dc 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -35,8 +35,9 @@ import ChatQuickActions from './ChatQuickActions.svelte' import ContextUsageIndicator from './ContextUsageIndicator.svelte' import AIChatModelSettings from './AIChatModelSettings.svelte' - import McpConnections from './McpConnections.svelte' - import SkillsPicker from './SkillsPicker.svelte' + import AssistantSettingsModal from './AssistantSettingsModal.svelte' + import { SkillsMenu } from './skills/skillsMenu.svelte' + import { McpMenu } from '$lib/components/mcp/mcpMenu.svelte' import ChatMode from './ChatMode.svelte' import DatatableCreationPolicy from './DatatableCreationPolicy.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' @@ -207,8 +208,11 @@ } = $props() let aiChatInput: AIChatInput | undefined = $state() - let mcpConnections: McpConnections | undefined = $state() - let skillsPicker: SkillsPicker | undefined = $state() + let assistantSettings: AssistantSettingsModal | undefined = $state() + // The "+" menu's skill and MCP rows: enough state to check and flip one, with + // everything else about them behind the assistant settings modal. + const skillsMenu = new SkillsMenu(aiChatManager, () => assistantSettings?.open('skills')) + const mcpMenu = new McpMenu(aiChatManager, () => assistantSettings?.open('mcp')) let plusMenuOpen = $state(false) let editingMessageIndex = $state(null) @@ -223,7 +227,15 @@ const active = document.activeElement const focusOnChat = !active || active === document.body || (panelEl?.contains(active) ?? false) - if (!focusOnChat) return + // An Escape while a run form is open must not discard what the user typed, so the action + // row alone stops the turn — wherever it is mounted, since the preview panel holds the + // form outside `panelEl`. Matched by call: two chats can be loading at once, and one's + // row must not answer for the other. + if (aiChatManager.hasPendingRunForm) { + const row = active?.closest('[data-run-form-actions]') + const toolCallId = row?.getAttribute('data-run-form-actions') + if (!toolCallId || !aiChatManager.isRunFormPending(toolCallId)) return + } else if (!focusOnChat) return e.preventDefault() // Immediate form: other chat panels' identical listeners must not // also cancel on body focus, nor a drawer/modal close on this press. @@ -555,7 +567,7 @@ const yoloBypassedTools = $derived.by(() => { return aiChatManager.tools - .filter((tool) => tool.requiresConfirmation === true) + .filter((tool) => tool.requiresConfirmation === true || tool.bypassedByAutoAccept === true) .map((tool) => ({ name: tool.def.function.name, // confirmationMessage may be a function of the call args, which we don't @@ -959,8 +971,8 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> const closeMenu = () => (plusMenuOpen = false) const inGlobal = aiChatManager.mode === AIMode.GLOBAL const [skillItems, mcpItems] = await Promise.all([ - inGlobal ? skillsPicker?.menuItems(closeMenu) : undefined, - inGlobal ? mcpConnections?.menuItems(closeMenu) : undefined + inGlobal ? skillsMenu.items(closeMenu) : undefined, + inGlobal ? mcpMenu.items(closeMenu) : undefined ]) return [ { @@ -1143,10 +1155,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/if} - + + {#if aiChatManager.mode === AIMode.GLOBAL} - - + {/if} {#if aiChatManager.mode === AIMode.APP && appContext && (appContext.inspectorElement || appContext.codeSelection)} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index fb54204fe9..aafb6a2ed1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -23,11 +23,14 @@ import { type ToolCallbacks, type ToolDisplayMessage, type UserQuestionDisplay, + type RunFormDisplay, + type RunFormDraft, type ChatJob, type ChatJobInit, type ChatJobStatus, completedJobToolStatus, backgroundJobCompletionNote, + createJobUpdateReader, deriveChatJobStatus, pendingToolImagesMessage, trimJob @@ -57,6 +60,7 @@ import { buildSummaryMessageContent } from './compactionPrompt' import { dfs } from '$lib/components/flows/previousResults' +import { redactFileArgs, redactSecretArgs } from '$lib/components/job_args' import { SvelteMap, SvelteSet } from 'svelte/reactivity' import { createLongHash } from '$lib/editorLangUtils' import type { AIProvider, UserDraftItemKind } from '$lib/gen' @@ -189,6 +193,22 @@ const MAX_CONSECUTIVE_COMPACTION_FAILURES = 3 // (panel teardown, save-and-clear) pass their own reason, so the queued-message // flush can tell "the user wants to move on" from "the turn was torn down". const USER_CANCEL_REASON = 'user_cancelled' +// Applied wherever a run form stops rendering. Only the form reads the deployed schema, +// so past that point it is a copy of the script's declarations — password and file +// defaults with them — persisted for the life of the chat. +const settledRunForm = (runForm: RunFormDisplay): RunFormDisplay => + runForm.submitted || runForm.canceled + ? { ...runForm, schema: undefined, code: undefined, lang: undefined } + : runForm + +/** A run form the chat is holding open, keyed by tool call id. */ +type PendingRunForm = { + /** Absent once the loop is no longer waiting: a card restored from history still mounts + * its form and still holds edits, but nothing is left to receive them. */ + resolve?: (args: Record | undefined) => void + draft: RunFormDraft + submitting: boolean +} // Built-in `/compact` session command — summarizes the conversation locally // instead of sending a turn to the model. Matched on the whole input so a // regular message that merely mentions "/compact" mid-sentence is unaffected. @@ -495,10 +515,32 @@ export class AIChatManager { // Consecutive getJob failures per background job, so a vanished/404 job can be // drained instead of polled forever. Ephemeral, keyed by jobId. #jobPollFailures = new Map() + // Incremental log/result-stream readers, keyed by jobId. A job that detaches out of + // the inline wait keeps streaming into its card through these; each holds its own + // offsets, so one created after a reload refetches from the start. + #jobUpdateReaders = new Map>() /** Opens a run in the sessions preview pane. Set by the session runtime; * undefined in the global side-panel chat, where the tray falls back to opening * the run in a new browser tab. */ openRunInPreview?: (a: { jobId: string; workspace: string; label: string }) => void + /** Opens a pending run form in the sessions preview pane, on the same tool call the + * chat card holds. Unset outside a session: a chat-bound form has nowhere else to go, + * so the card hides the control rather than offering a tab that cannot run. */ + openRunForm?: (a: { toolCallId: string; label: string }) => void + closeRunForm?: (toolCallId: string) => void + /** Hands that tab from the form to the run it just started, in place: the tab keeps its + * position in the strip and stays active if it was. */ + showRunInPlaceOfForm?: (a: { + toolCallId: string + jobId: string + workspace: string + label: string + }) => void + /** Whether the panel holds this call's pending form. Answered off the session's tab list, + * so it stays true while the user is on another tab, and per call rather than "the open + * one". Read from a `$derived` — the reader subscribes to the tab list through the call. + * The card hides its form on it, which is what keeps exactly one mounted per call. */ + isRunFormInPreview?: (toolCallId: string) => boolean openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void closeArtifact?: (artifactId: string) => void #loading = $state(false) @@ -663,7 +705,13 @@ export class AIChatManager { >(undefined) scriptEditorShowDiffMode = $state<(() => void) | undefined>(undefined) scriptEditorGetLintErrors = $state<(() => ScriptLintResult) | undefined>(undefined) + /** The editor a FLOW-mode chat belongs to: the page owning the chat names itself here, and a + * nested editor (a subflow drawer) takes it over while it is open. Unset in a session chat, + * which keeps every open editor tab mounted and could only name an arbitrary one — a session + * resolves an editor by its storage path through `flowEditorFor`. */ flowAiChatHelpers = $state(undefined) + /** Every mounted flow editor. */ + #flowEditors = new Set() appAiChatHelpers = $state(undefined) /** Datatable creation policy: enabled flag, datatable name, and optional schema */ datatableCreationPolicy = $state<{ @@ -682,6 +730,15 @@ export class AIChatManager { { resolve: (value: boolean) => void; toolName?: string } >() private userQuestionCallbacks = new Map void>() + /** + * One run form's whole life while it waits, so ending it is one delete and cannot end half + * of it. Held here rather than in the form, which unmounts and remounts as it moves between + * the chat card and the preview pane: both are views of one entry. + * + * Entries are replaced rather than mutated, so a `$derived` reading `submitting` fires; + * `draft` keeps its identity across a replacement, which is what the form is bound to. + */ + #runForms = new SvelteMap() private appDatatablesRefreshTimeout: ReturnType | undefined = undefined disabledModes: Partial> = $state({}) @@ -806,19 +863,23 @@ export class AIChatManager { // turn-end save. #maskPersistQueue: Promise = Promise.resolve() #persistModifiedItems(): Promise { - this.#maskPersistQueue = this.#maskPersistQueue.then(() => - this.historyManager - .saveChat( - this.displayMessages, - this.messages, - this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined - ) - // Swallow (and log) a failed write so it can't wedge the queue as a - // rejected link — the next persist snapshots the full current set, so - // a lost write self-heals on the next mutation or turn-end save. - .catch((e) => console.error('Failed to persist modified-items mask', e)) - ) + this.#maskPersistQueue = this.#maskPersistQueue.then(() => { + const { display, jobs } = this.#interruptedSnapshot() + return ( + this.historyManager + .saveChat( + display, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined, + jobs + ) + // Swallow (and log) a failed write so it can't wedge the queue as a + // rejected link — the next persist snapshots the full current set, so + // a lost write self-heals on the next mutation or turn-end save. + .catch((e) => console.error('Failed to persist modified-items mask', e)) + ) + }) return this.#maskPersistQueue } @@ -849,6 +910,16 @@ export class AIChatManager { ...this.backgroundJobs, { ...init, createdAt: Date.now(), status: 'queued', detached: false, reported: false } ] + // The panel was holding this call's form and the call now has a job: the tab follows + // the call rather than being left on a form that has already run. + if (this.isRunFormInPreview?.(init.toolCallId)) { + this.showRunInPlaceOfForm?.({ + toolCallId: init.toolCallId, + jobId: init.jobId, + workspace: init.workspace, + label: init.label + }) + } } /** Merge a partial update into a tracked job by id. */ @@ -962,10 +1033,34 @@ export class AIChatManager { let anyTerminal = false for (const job of pending) { try { + // Its own output first, so a run that detached out of the inline wait keeps + // filling its card. `getJob` alone would freeze a streamed result until the + // job landed — the partial is only on the updates endpoint. + let reader = this.#jobUpdateReaders.get(job.jobId) + if (!reader) { + reader = createJobUpdateReader(job.jobId, job.workspace) + this.#jobUpdateReaders.set(job.jobId, reader) + } + const update = await reader.poll() + if (gen !== this.#jobPollGeneration) return + // Only what this reader has collected: the patch is spread over the card, so + // naming a field it has nothing for erases output already on it. + if (update?.logs || update?.resultStream) { + this.applyToolStatus(job.toolCallId, { + ...(update.logs ? { logs: update.logs } : {}), + ...(update.resultStream ? { resultStream: update.resultStream } : {}) + }) + } + + // Only when the reader has not already carried them, or when the run may be + // over — the tail written between the last poll and the end is on the job + // alone. Otherwise these are logs the tray strips and the card already has, + // fetched a second time every tick, for every detached job in the chat. + const wantLogs = !update || update.completed const fetched = await JobService.getJob({ workspace: job.workspace, id: job.jobId, - noLogs: false, + noLogs: !wantLogs, noCode: true }) // The user switched conversations while this getJob was in flight; its @@ -975,7 +1070,19 @@ export class AIChatManager { this.#jobPollFailures.delete(job.jobId) if (fetched.type === 'CompletedJob') { anyTerminal = true - this.#onBackgroundJobComplete(job, fetched as CompletedJob) + this.#jobUpdateReaders.delete(job.jobId) + // The updates can call a landed job unfinished, and the model reads these + // logs, so a completion seen without them is fetched again. + const completed = wantLogs + ? (fetched as CompletedJob) + : ((await JobService.getJob({ + workspace: job.workspace, + id: job.jobId, + noLogs: false, + noCode: true + })) as CompletedJob) + if (gen !== this.#jobPollGeneration) return + this.#onBackgroundJobComplete(job, completed) } else { // Store the derived status and the trimmed Job together so the tray // badge (JobStatusIcon) and the scalar status can never drift. @@ -996,6 +1103,7 @@ export class AIChatManager { this.#jobPollFailures.set(job.jobId, failures) if (httpStatus === 404 || failures >= 5) { this.#jobPollFailures.delete(job.jobId) + this.#jobUpdateReaders.delete(job.jobId) // Vanished (404) or unreachable after repeated polls. Mark it failed WITH // a snapshot + tool-card patch (mirroring #onBackgroundJobComplete) so // neither the tray badge nor the launching tool card stays frozen on @@ -1013,7 +1121,8 @@ export class AIChatManager { this.updateJob(job.jobId, { status: 'failure', reported: true, job: trimJob(gone) }) this.applyToolStatus(job.toolCallId, { content: 'Background job could not be retrieved (it may have been removed)', - error: `Job ${job.jobId} was unreachable` + error: `Job ${job.jobId} was unreachable`, + isLoading: false }) anyTerminal = true } else { @@ -1055,8 +1164,14 @@ export class AIChatManager { status === 'canceled' || !job.resultFormat ? undefined : formatChatJobCompletion(completed, job.resultFormat) - // Fill the tool card that launched it (we run outside a turn here). - this.applyToolStatus(job.toolCallId, formatted?.card ?? completedJobToolStatus(completed)) + // Fill the tool card that launched it (we run outside a turn here). isLoading is + // normally already false — processToolCall clears it when the launching tool + // returns — but a card restored from a mid-turn checkpoint never saw that return, + // so only this patch can stop it spinning. + this.applyToolStatus(job.toolCallId, { + ...(formatted?.card ?? completedJobToolStatus(completed)), + isLoading: false + }) // A user-canceled job needs no model note or auto-resume: the user stopped it // deliberately, so announcing it (as "FAILED", since a canceled job isn't a // success) or burning a turn on it would be noise. @@ -1130,17 +1245,12 @@ export class AIChatManager { // (saveChat keeps the prior mask when it is undefined). #jobPersistQueue: Promise = Promise.resolve() #persistBackgroundJobs(): Promise { - this.#jobPersistQueue = this.#jobPersistQueue.then(() => - this.historyManager - .saveChat( - this.displayMessages, - this.messages, - this.contextUsage, - undefined, - $state.snapshot(this.backgroundJobs) - ) + this.#jobPersistQueue = this.#jobPersistQueue.then(() => { + const { display, jobs } = this.#interruptedSnapshot() + return this.historyManager + .saveChat(display, this.messages, this.contextUsage, undefined, jobs) .catch((e) => console.error('Failed to persist background jobs', e)) - ) + }) return this.#jobPersistQueue } @@ -1152,6 +1262,7 @@ export class AIChatManager { this.#jobPollGeneration++ clearTimeout(this.#autoResumeRetry) this.#autoResumeRetry = undefined + this.#jobUpdateReaders.clear() this.backgroundJobs = [] this.pendingJobNotes = [] } @@ -1787,6 +1898,166 @@ export class AIChatManager { return true } + requestRunArgs = ( + toolId: string, + form: RunFormDisplay, + opts?: { autoAccepted?: boolean } + ): Promise | undefined> => { + // The tool reads the schema before it asks, so a stop during that read drains the + // callbacks and settles the card before this runs. Installing one then would park + // the turn on a form the settled card no longer renders, leaving nothing able to + // resolve it. The controller is per-turn, so a later turn still opens. + if (this.abortController?.signal.aborted) { + // Settle the form the tool attached after the stop. Its card is about to stop + // loading without ever having rendered, and settledToolDisplay only reaches a + // loading one — so this is the last point the schema, with the script's own + // password and file defaults, can be dropped. No card copy: the stop path writes + // what the row says. + this.#settleRunForm(toolId, undefined) + return Promise.resolve(undefined) + } + // Ahead of the wait, not of the stop above: the caller settled this form before + // attaching it, so its card renders no fields and nothing here could ever resolve. + if (opts?.autoAccepted) { + return Promise.resolve(form.args) + } + // Seeded from the caller's copy, before the card renders: the file arguments on + // `displayMessages` are redacted, so a draft built from those would open the form on + // the marker rather than on the bytes the model proposed. + const entry = this.#runFormEntry(toolId, form) + return new Promise((resolve) => { + this.#runForms.set(toolId, { ...entry, resolve }) + }) + } + + /** + * The entry a run form edits through, created on first mount and shared by every later one. + * + * Deep snapshots, never the message's own values: those are `$state` proxies off + * `displayMessages`, and SchemaForm edits args and schema in place (it reorders the + * schema on mount), so anything shallower writes each keystroke — a nested password + * included — into the persisted transcript. + */ + #runFormEntry = (toolId: string, runForm: RunFormDisplay): PendingRunForm => { + const existing = this.#runForms.get(toolId) + if (existing) return existing + const draft = $state({ + args: ($state.snapshot(runForm.args) ?? {}) as Record, + schema: ($state.snapshot(runForm.schema) ?? {}) as Record + }) + const entry: PendingRunForm = { draft, submitting: false } + this.#runForms.set(toolId, entry) + return entry + } + + runFormDraft = (toolId: string, runForm: RunFormDisplay): RunFormDraft => + this.#runFormEntry(toolId, runForm).draft + + markRunFormStarted = (toolId: string) => this.#patchRunForm(toolId, { started: true }) + + // A form restored from history has an entry once it mounts, but no resolve: the loop + // that opened it is gone. + isRunFormPending = (toolId: string): boolean => !!this.#runForms.get(toolId)?.resolve + + isRunFormSubmitting = (toolId: string): boolean => this.#runForms.get(toolId)?.submitting ?? false + + /** False when a submit is already in flight for this call, so the caller can drop a + * second one rather than mint a second set of ephemeral secret variables for it. */ + beginRunFormSubmit = (toolId: string): boolean => { + const entry = this.#runForms.get(toolId) + if (!entry || entry.submitting) return false + this.#runForms.set(toolId, { ...entry, submitting: true }) + return true + } + + endRunFormSubmit = (toolId: string) => { + const entry = this.#runForms.get(toolId) + if (entry?.submitting) this.#runForms.set(toolId, { ...entry, submitting: false }) + } + + /** Whether any form of this chat is still waiting on the user. Asked instead of looking + * the form up in the panel's DOM: when the preview holds it, the card is collapsed and + * the only mounted copy is outside the panel — where a DOM query would miss it and let + * Escape discard what has been typed. */ + get hasPendingRunForm(): boolean { + for (const entry of this.#runForms.values()) if (entry.resolve) return true + return false + } + + /** False when the form is no longer pending, so the caller can say so instead of + * leaving its submit button spinning on a run that will never start. */ + handleRunFormSubmit = (toolId: string, args: Record): boolean => { + if (!this.isRunFormPending(toolId)) return false + this.#settleRunForm(toolId, args) + return true + } + + handleRunFormCancel = (toolId: string) => { + // The card's own copy is settled here rather than only in the tool's fn, which a form + // restored from history no longer has: Cancel is that card's one way out, and while it + // stays active the whole session reads as needs-confirmation (getSessionChatStatus asks + // pendingUserAction before loading). Clearing isLoading is part of settling — canceled + // alone unmounts the form but leaves the card shimmering. + this.#settleRunForm(toolId, undefined, (runForm) => ({ + isLoading: false, + error: 'Cancelled by user', + content: `Run of "${runForm.path}" cancelled by user` + })) + } + + /** + * The one way a run form stops waiting on the user: `submitted` is the arguments to run + * with, `undefined` a cancellation. + * + * `card` is for a settler that also owns what the row reads — pressing Cancel does, a + * stopped turn leaves it to settledToolDisplay. + */ + #settleRunForm = ( + toolId: string, + submitted: Record | undefined, + card?: (runForm: RunFormDisplay) => Partial + ) => { + const entry = this.#runForms.get(toolId) + // Its draft holds whatever was typed into the form, a minted password included. + this.#runForms.delete(toolId) + // Cancelled, so no run follows it into that tab (a submitted one is handed over by + // registerJob instead) — take the tab with it rather than leaving a dead form open. + if (submitted === undefined) this.closeRunForm?.(toolId) + const cancelledArgs = + submitted === undefined && entry ? this.#settledFormArgs(entry) : undefined + this.#patchRunForm( + toolId, + submitted ? { submitted: true } : { canceled: true }, + cancelledArgs ? (runForm) => ({ ...card?.(runForm), parameters: cancelledArgs }) : card + ) + entry?.resolve?.(submitted) + } + + /** + * What a form that never ran leaves on its card. A run writes its own arguments there once + * it has them and a cancellation never reaches that write, so without this the card keeps + * the proposal it was published on — naming a secret the field had already replaced with a + * reference. A reference stands, anything still literal does not. + */ + #settledFormArgs = (entry: PendingRunForm): Record => + redactFileArgs(redactSecretArgs(entry.draft.args, entry.draft.schema), entry.draft.schema) + + #patchRunForm = ( + toolId: string, + patch: Partial, + card?: (runForm: RunFormDisplay) => Partial + ) => { + this.displayMessages = this.displayMessages.map((message) => + message.role === 'tool' && message.tool_call_id === toolId && message.runForm + ? { + ...message, + ...card?.(message.runForm), + runForm: settledRunForm({ ...message.runForm, ...patch }) + } + : message + ) + } + setAiChatInput(aiChatInput: AIChatInput | null) { this.aiChatInput = aiChatInput } @@ -2123,7 +2394,10 @@ export class AIChatManager { // pipeline surface when a /pipeline editor has registered helpers. Centralized // so changeMode, refreshGlobalSkills, and setPipelineHelpers stay consistent — // each rebuild would otherwise drop the pipeline augmentation the others added. - private configureGlobalMode = () => { + // + // Public because it is purely local, unlike `changeMode(GLOBAL)`, which also + // fires the three network refreshes. + configureGlobalMode = () => { const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { previewTools: this.isSessionChat, user: this.globalIdentity, @@ -2147,7 +2421,8 @@ export class AIChatManager { openArtifact: this.openArtifact } : {}), - testActiveFlow: async (args?: Record) => this.flowAiChatHelpers?.testFlow(args), + testActiveFlow: async (storagePath: string, args?: Record) => + this.flowEditorFor(storagePath)?.testFlow(args), getModifiedItems: () => (this.modifiedItems ? [...this.modifiedItems] : undefined), attachedFiles: this.attachedFiles, getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', @@ -3308,7 +3583,7 @@ export class AIChatManager { ) if (messages.length === this.messages.length) return checkpointedShape = shape - const display = this.settledToolDisplay(this.displayMessages, 'Interrupted') + const { display, jobs } = this.#interruptedSnapshot() // onMessageEnd is what gives streamed text its bubble, and it clears // currentReply doing so — text still there has none, and without one the // reply returns as context the reader cannot see. @@ -3328,7 +3603,8 @@ export class AIChatManager { // partial turn — enough to skip the compaction its next send needs. // Omitting drops the field, which is the "readers estimate" fallback. undefined, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + jobs ) } catch (e) { console.error('Failed to checkpoint chat mid-turn', e) @@ -3698,6 +3974,8 @@ export class AIChatManager { isPlanModeActive: () => this.planModeActive, onToolBlockedByPlanMode: this.planMode.noteBlockedTool, requestUserQuestion: this.requestUserQuestion, + requestRunArgs: this.requestRunArgs, + markRunFormStarted: this.markRunFormStarted, onItemModified: (kind, path) => this.recordModifiedItem(kind, path), onItemDeployed: (kind, from, to) => void this.renameModifiedItem(kind, from, to), onItemDiscarded: (kind, path) => void this.removeModifiedItem(kind, path), @@ -3954,6 +4232,19 @@ export class AIChatManager { resolveQuestion(undefined) } this.userQuestionCallbacks.clear() + for (const [toolId, entry] of this.#runForms) { + entry.resolve?.(undefined) + // Stopping the turn is the form's other way out, and the draft dies with this loop: + // settledToolDisplay settles the card below without ever seeing what was typed. + this.#patchRunForm(toolId, {}, () => ({ parameters: this.#settledFormArgs(entry) })) + // The form settles with the turn, so a preview tab holding it goes too rather + // than being left on a form that can no longer run. + this.closeRunForm?.(toolId) + } + // Not through #settleRunForm: settledToolDisplay settles every card of the stopped + // turn at once, and it alone can tell a run that reached the server from one that + // never did. + this.#runForms.clear() const cancelReason = reason ?? USER_CANCEL_REASON console.log('cancelling request:', { reason: cancelReason, @@ -4190,6 +4481,15 @@ export class AIChatManager { if (this.isJobNonTerminal(j.status)) j.detached = true } if (this.backgroundJobs.length > 0) this.backgroundJobs = [...this.backgroundJobs] + // Reloading resolves no card on its own. Settle every one the poller above + // will not reach, whoever wrote it — a record from a build that stored cards + // without their jobs would otherwise restore one that spins forever. + const pollable = this.#pollableToolCalls() + this.displayMessages = this.settledToolDisplay( + this.displayMessages, + 'Interrupted', + (message) => !pollable.has(message.tool_call_id) + ) this.#ensureJobPoller() // Message-attached files live in the transcript, not in the store's // persistence — rebuild their rows so the loaded chat's references are @@ -4371,7 +4671,11 @@ export class AIChatManager { } setFlowHelpers = (flowHelpers: FlowAIChatHelpers) => { - this.flowAiChatHelpers = flowHelpers + this.#flowEditors.add(flowHelpers) + // Only a chat that can reach FLOW mode names an editor (see `flowAiChatHelpers`). + if (!this.isSessionChat) { + this.flowAiChatHelpers = flowHelpers + } untrack(() => { if (this.autoAcceptEditsActive) { this.acceptPendingFlowEdits(flowHelpers) @@ -4379,10 +4683,17 @@ export class AIChatManager { }) return () => { - this.flowAiChatHelpers = undefined + this.#flowEditors.delete(flowHelpers) + if (!this.isSessionChat) { + this.flowAiChatHelpers = undefined + } } } + private flowEditorFor(storagePath: string): FlowAIChatHelpers | undefined { + return [...this.#flowEditors].find((helpers) => helpers.getStoragePath() === storagePath) + } + // Registered by the /pipeline editor while it is mounted. Rebuilds the global // tool set so the pipeline tools appear (and disappear on unregister). Pipeline // AI edits apply directly as drafts, so there is nothing to auto-accept. @@ -4519,10 +4830,25 @@ export class AIChatManager { // through here first. private settledToolDisplay = ( messages: DisplayMessage[], - messageText: string + messageText: string, + shouldSettle: (message: ToolDisplayMessage) => boolean = () => true ): DisplayMessage[] => messages.map((message) => { - if (message.role === 'tool' && (message.isLoading || message.isQueued)) { + if ( + message.role === 'tool' && + (message.isLoading || message.isQueued) && + shouldSettle(message) + ) { + // Stopping the turn does not stop the job, and between Run and the job's id + // there is no way to know whether the server queued one: nothing threads the + // abort into that request, so it lands either way. That window says so + // rather than picking a side — "canceled" hides a script that ran, "started" + // invents one that did not. + const runState = message.runForm?.started + ? 'started' + : message.runForm?.submitted + ? 'starting' + : 'idle' return { ...message, isLoading: false, @@ -4532,14 +4858,25 @@ export class AIChatManager { // and a card that hides its result as still-streaming. needsConfirmation: false, isStreamingArguments: false, - // A question's card disappears once canceled, so keep the question - // itself readable in the collapsed header. + // An interactive card disappears once canceled, so keep what it was + // asking readable in the collapsed header. content: message.userQuestion ? `Asked: ${message.userQuestion.question} — ${messageText}` - : messageText, - error: messageText, + : message.runForm + ? runState === 'started' + ? `Run ${message.runForm.path} — started, stopped tracking before it finished` + : runState === 'starting' + ? `Run ${message.runForm.path} — ${messageText} while starting, check the runs page for a job` + : `Run ${message.runForm.path} — ${messageText}` + : messageText, + // A run that reached the server keeps whatever the job reported: it is not + // this turn's error, and the jobs tray is still following it. + ...(runState === 'idle' ? { error: messageText } : {}), userQuestion: message.userQuestion ? { ...message.userQuestion, canceled: true } + : undefined, + runForm: message.runForm + ? settledRunForm({ ...message.runForm, canceled: runState === 'idle' }) : undefined } } @@ -4549,6 +4886,36 @@ export class AIChatManager { cancelLoadingTools = (messageText: 'Canceled' | 'Error' = 'Canceled') => { this.displayMessages = this.settledToolDisplay(this.displayMessages, messageText) } + + /** What the transcript would be if the turn stopped here — for the writes that fire + * mid-turn without ending it. Loading is a property of this page: reloading resolves no + * card, so one stored still pending comes back asking for input nothing can deliver. + * Settles the stored copy only; the live turn keeps its cards. + * + * Except a card the poller will resolve after a reload: settling that one stores an + * "Interrupted" error the patch a completed job merges in carries nothing to clear. + * Which cards those are is loadPastChat's question, asked the same way — and the poller + * only knows the jobs stored in the same record, so both go into the same saveChat. */ + #interruptedSnapshot = (): { display: DisplayMessage[]; jobs: ChatJob[] } => { + const polled = this.#pollableToolCalls() + return { + display: this.settledToolDisplay( + this.displayMessages, + 'Interrupted', + (message) => !polled.has(message.tool_call_id) + ), + jobs: $state.snapshot(this.backgroundJobs) as ChatJob[] + } + } + + /** Tool calls a restored transcript can still resolve. loadPastChat re-attaches the + * poller to every non-terminal job and nothing else runs after a reload, so this is + * the whole set — asked identically when storing a card and when restoring one, or + * the two drift and a card is kept by one and stranded by the other. */ + #pollableToolCalls = (): Set => + new Set( + this.backgroundJobs.filter((j) => this.isJobNonTerminal(j.status)).map((j) => j.toolCallId) + ) } export const aiChatManager = new AIChatManager() diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 084dc2c040..7142646d90 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { writable } from 'svelte/store' import type { FlowAIChatHelpers } from './flow/core' import type { PipelineAIChatHelpers } from './pipeline/core' import type { CurrentEditor } from '$lib/components/flows/types' @@ -36,6 +37,7 @@ const mocks = vi.hoisted(() => ({ runChatLoop: vi.fn(), listResource: vi.fn(), getJob: vi.fn(), + getJobUpdates: vi.fn(), whoami: vi.fn(), workspace: 'test_workspace' as string | undefined, // The workspace being browsed, which a session chat's own workspace need not be. @@ -60,7 +62,8 @@ vi.mock('$lib/gen', () => ({ whoami: mocks.whoami }, JobService: { - getJob: mocks.getJob + getJob: mocks.getJob, + getJobUpdates: mocks.getJobUpdates } })) @@ -121,6 +124,9 @@ vi.mock('$lib/toast', () => ({ })) vi.mock('$lib/aiStore', () => ({ + // `sendRequest` reads it before anything else, so a test that goes through a real turn + // rather than driving the manager directly needs it present and enabled. + copilotInfo: writable({ enabled: true, workspaceDisabled: false, aiModels: [] }), getCurrentModel: mocks.getCurrentModel, tryGetCurrentModel: mocks.tryGetCurrentModel, getCombinedCustomPrompt: () => '', @@ -172,6 +178,10 @@ beforeEach(() => { mocks.getOpenaiClient.mockReturnValue({}) mocks.getAnthropicClient.mockReturnValue({}) mocks.listResource.mockResolvedValue([]) + // Re-seeded here rather than in the factory: clearAllMocks keeps implementations, so a + // test that makes the updates endpoint fail would otherwise leave it failing for the rest + // of the file. Neutral by default — completion is getJob's answer. + mocks.getJobUpdates.mockResolvedValue({ completed: false, running: true }) mocks.workspace = 'test_workspace' mocks.runChatLoop.mockResolvedValue({ addedMessages: [], @@ -181,15 +191,18 @@ beforeEach(() => { }) function createFlowHelpers({ - hasPendingChanges, - acceptAllModuleActions, - testFlow = vi.fn() + hasPendingChanges = () => false, + acceptAllModuleActions = vi.fn(), + testFlow = vi.fn(), + storagePath = 'u/admin/live_flow' }: { - hasPendingChanges: () => boolean - acceptAllModuleActions: () => void + hasPendingChanges?: () => boolean + acceptAllModuleActions?: () => void testFlow?: FlowAIChatHelpers['testFlow'] -}): FlowAIChatHelpers { + storagePath?: string +} = {}): FlowAIChatHelpers { return { + getStoragePath: () => storagePath, getFlowAndSelectedId: vi.fn(), getRootModules: vi.fn(), inlineScriptSession: { get: vi.fn(), set: vi.fn(), clear: vi.fn() }, @@ -226,6 +239,241 @@ describe('AIChatManager unmounted-chat guard', () => { }) }) +describe('AIChatManager run form', () => { + // A transcript can be persisted mid-turn (a background job's status write) and + // restored into a fresh manager, which has none of the turn's callbacks. Cancel is + // then the card's only exit, and until it settles pendingUserAction keeps the whole + // session reading as needs-confirmation. + // A save that fires mid-turn (jobs tray, review dock) stores a transcript nothing + // will resume. Storing a card still pending brings back a form whose Run resolves + // no callback. + it('stores loading cards settled when a mid-turn save fires', async () => { + const manager = new AIChatManager() + manager.displayMessages = [ + { + role: 'tool', + tool_call_id: 'call_r', + content: 'Waiting for you to confirm the arguments of "f/a/b"', + isLoading: true, + runForm: { path: 'f/a/b', schema: {}, args: {} } + } + ] + const saveChat = vi.spyOn(manager.historyManager, 'saveChat').mockResolvedValue(undefined) + + manager.markJobsReviewed([]) + manager.dismissJob('nope') + await Promise.resolve() + + const { isActiveRunForm } = await import('./shared') + const stored = saveChat.mock.calls.at(-1)?.[0]?.[0] + expect(stored?.runForm?.canceled).toBe(true) + // What every mid-turn save has to hold: no stored card renders a live form. A + // save path added without settling would restore a Run that resolves nothing. + expect(isActiveRunForm(stored!)).toBe(false) + // The live card is untouched — the turn is still parked on it. + expect(manager.displayMessages[0].isLoading).toBe(true) + }) + + // Only the rendered form reads the schema, and a settled card renders none. Kept, it + // would sit in history for the life of the chat with the script's own password and + // file defaults inside it. + it('drops the schema from a card that has stopped showing a form', () => { + const manager = new AIChatManager() + const runForm = { path: 'f/a/b', schema: { properties: { tok: { password: true } } }, args: {} } + manager.displayMessages = [ + { role: 'tool', tool_call_id: 'call_r', content: '', isLoading: true, runForm } + ] + + manager.handleRunFormCancel('call_r') + + expect(manager.displayMessages[0].runForm?.schema).toBeUndefined() + expect(manager.displayMessages[0].runForm?.canceled).toBe(true) + }) + + // A run writes what it ran onto the card; a cancelled one never gets there, so without + // this its Inputs tab still names the proposal the card was published on — a secret the + // mounted field had already replaced with a reference. + it('settles a cancelled card on what the form held, not on the proposal', async () => { + const manager = new AIChatManager() + const schema = { + properties: { token: { password: true }, spare: { password: true }, note: {} } + } + const runForm = { + path: 'f/a/b', + schema, + args: { token: 'hunter2', spare: 'untouched', note: 'hello' } + } + manager.displayMessages = [ + { + role: 'tool', + tool_call_id: 'call_r', + content: '', + isLoading: true, + parameters: { ...runForm.args }, + runForm + } + ] + + void manager.requestRunArgs('call_r', runForm) + const draft = manager.runFormDraft('call_r', runForm) + draft.args.token = '$var:u/admin/secret_arg/AbC' + draft.args.note = 'goodbye' + + manager.handleRunFormCancel('call_r') + + expect(manager.displayMessages[0].parameters).toEqual({ + token: '$var:u/admin/secret_arg/AbC', + // Never minted, so still the secret itself. + spare: '', + note: 'goodbye' + }) + }) + + // Stopping the turn is the form's other way out, and it settles cards through + // settledToolDisplay rather than through #settleRunForm. + it('settles a stopped form on what it held too', () => { + const manager = new AIChatManager() + const schema = { properties: { token: { password: true }, note: {} } } + const runForm = { path: 'f/a/b', schema, args: { token: 'hunter2', note: 'hello' } } + manager.displayMessages = [ + { + role: 'tool', + tool_call_id: 'call_r', + content: '', + isLoading: true, + parameters: { ...runForm.args }, + runForm + } + ] + + void manager.requestRunArgs('call_r', runForm) + const draft = manager.runFormDraft('call_r', runForm) + draft.args.token = '$var:u/admin/secret_arg/AbC' + draft.args.note = 'goodbye' + + manager.cancel() + + expect(manager.displayMessages[0].parameters).toEqual({ + token: '$var:u/admin/secret_arg/AbC', + note: 'goodbye' + }) + }) + + // The tool reads the deployed schema before it asks for arguments. A stop during that + // read drains the callbacks and settles the card, so a waiter installed afterwards was + // one no rendered form could resolve: the turn stayed loading until a second stop. + it('installs no run-form waiter once the turn is stopped', async () => { + const manager = new AIChatManager() + // The turn the tool is running under; cancel aborts it. + manager.abortController = new AbortController() + manager.displayMessages = [ + { + role: 'tool', + tool_call_id: 'call_late', + content: 'Executing...', + isLoading: true + } + ] + + manager.cancel() + + await expect( + manager.requestRunArgs('call_late', { path: 'f/a/b', schema: {}, args: {} }) + ).resolves.toBeUndefined() + expect(manager.isRunFormPending('call_late')).toBe(false) + }) + + // The stop lands while the tool is still reading the deployed schema, so the form is + // attached after the card was settled. Nothing settles it a second time — the card + // stops loading without the form ever rendering — so the schema would otherwise stay + // in the transcript with the script's own password default inside it. + it('drops the schema from a form attached after the turn was stopped', async () => { + const manager = new AIChatManager() + manager.abortController = new AbortController() + manager.displayMessages = [ + { role: 'tool', tool_call_id: 'call_x', content: 'Executing...', isLoading: true } + ] + manager.cancel() + + const runForm = { + path: 'f/a/b', + schema: { properties: { tok: { password: true, default: 'hunter2' } } }, + args: {} + } + manager.applyToolStatus('call_x', { content: 'Waiting for you...', runForm, isLoading: true }) + + await expect(manager.requestRunArgs('call_x', runForm)).resolves.toBeUndefined() + expect(manager.displayMessages[0].runForm?.schema).toBeUndefined() + expect(JSON.stringify(manager.displayMessages[0])).not.toContain('hunter2') + }) + + // Stop ends the turn, not the job: the deployed script is already running with all + // its side effects, so the transcript must not record it as cancelled. + it('does not mark a started run cancelled when the turn is stopped', () => { + const manager = new AIChatManager() + manager.displayMessages = [ + { + role: 'tool', + tool_call_id: 'call_s', + content: 'Running "f/a/b"...', + isLoading: true, + runForm: { path: 'f/a/b', schema: {}, args: {}, submitted: true, started: true } + } + ] + + manager.cancelLoadingTools() + + const settled = manager.displayMessages[0] + expect(settled.runForm?.canceled).toBe(false) + expect(settled.error).toBe(undefined) + expect(settled.isLoading).toBe(false) + }) + + // Run flips `submitted` a round trip before the job id arrives, and nothing threads + // the stop into that request — so the card claims neither outcome for that window. + it('claims neither outcome for a run stopped while its job was starting', () => { + const manager = new AIChatManager() + manager.displayMessages = [ + { + role: 'tool', + tool_call_id: 'call_s', + content: 'Running "f/a/b"...', + isLoading: true, + runForm: { path: 'f/a/b', schema: {}, args: {}, submitted: true } + } + ] + + manager.cancelLoadingTools() + + const settled = manager.displayMessages[0] + expect(settled.runForm?.canceled).toBe(false) + expect(settled.error).toBe(undefined) + expect(settled.content).toBe( + 'Run f/a/b — Canceled while starting, check the runs page for a job' + ) + }) + + // Only a form the user never submitted was cancelled outright. + it('marks an unsubmitted run cancelled when the turn is stopped', () => { + const manager = new AIChatManager() + manager.displayMessages = [ + { + role: 'tool', + tool_call_id: 'call_u', + content: 'Waiting for you to confirm the arguments of "f/a/b"', + isLoading: true, + runForm: { path: 'f/a/b', schema: {}, args: {} } + } + ] + + manager.cancelLoadingTools() + + const settled = manager.displayMessages[0] + expect(settled.runForm?.canceled).toBe(true) + expect(settled.content).toBe('Run f/a/b — Canceled') + }) +}) + describe('AIChatManager.sendOrQueue', () => { // The programmatic senders (an editor's "AI Fix", an arriving hand-off) have no // composer to enforce the composer's rule for them: a second loop on one manager @@ -332,11 +580,11 @@ describe('AIChatManager global skills', () => { mocks.tryGetCurrentModel.mockReturnValue(model) }) - // Only selected skills reach the prompt, and the selection is keyed by - // workspace and account (see skills/enabledSkills.ts). - function selectSkills(workspace: string, ...paths: string[]) { + // Every readable skill reaches the prompt; only the paths someone decided about + // are stored, keyed by workspace and account (see skills/enabledSkills.ts). + function turnOffSkills(workspace: string, ...paths: string[]) { const stored = JSON.parse(localStorage.getItem('wm_skills_enabled') ?? '{}') - stored[`${workspace}:${TEST_EMAIL}`] = paths + stored[`${workspace}:${TEST_EMAIL}`] = Object.fromEntries(paths.map((p) => [p, false])) localStorage.setItem('wm_skills_enabled', JSON.stringify(stored)) } @@ -346,8 +594,6 @@ describe('AIChatManager global skills', () => { resolveParentSkills = resolve }) mocks.workspace = 'parent' - selectSkills('parent', 'f/skills/parent-skill') - selectSkills('child', 'f/skills/child-skill') mocks.listResource.mockImplementation(({ workspace }: { workspace: string }) => { if (workspace === 'parent') { return parentSkills @@ -391,12 +637,12 @@ describe('AIChatManager global skills', () => { expect(manager.systemMessage.content).not.toContain('parent-skill') }) - it('leaves a readable but unselected skill out of the prompt', async () => { + it('leaves a skill turned off out of the prompt', async () => { mocks.listResource.mockResolvedValue([ - { path: 'f/skills/selected', description: 'the one turned on' }, - { path: 'f/skills/unselected', description: 'readable but never turned on' } + { path: 'f/skills/selected', description: 'left on, like every skill starts' }, + { path: 'f/skills/unselected', description: 'the one turned off' } ]) - selectSkills('test_workspace', 'f/skills/selected') + turnOffSkills('test_workspace', 'f/skills/unselected') const manager = new AIChatManager() manager.isSessionChat = true @@ -411,7 +657,6 @@ describe('AIChatManager global skills', () => { mocks.listResource.mockResolvedValue([ { path: 'u/admin/review-code', description: 'review code for bugs' } ]) - selectSkills('test_workspace', 'u/admin/review-code') mocks.runChatLoop.mockImplementation(async (config: any) => { const userMessage = config.messages[config.messages.length - 1] expect(userMessage.content).toContain('Use the skill at "u/admin/review-code". find bugs') @@ -438,7 +683,6 @@ describe('AIChatManager global skills', () => { { path: 'u/admin/deploy', description: 'personal deploy steps' }, { path: 'f/team/deploy', description: 'the team deploy steps' } ]) - selectSkills('test_workspace', 'u/admin/deploy', 'f/team/deploy') mocks.runChatLoop.mockImplementation(async (config: any) => { // Picking either one would silently apply instructions the user did not // choose, so the text is left alone for the model to ask about. @@ -608,19 +852,36 @@ describe('AIChatManager autonomy mode', () => { manager.isSessionChat = true manager.sessionId = 'htc1xouxd96dcyo6ruqo39' - manager.setFlowHelpers( - createFlowHelpers({ - hasPendingChanges: () => false, - acceptAllModuleActions: vi.fn(), - testFlow - }) - ) + manager.setFlowHelpers(createFlowHelpers({ testFlow })) manager.changeMode(AIMode.GLOBAL) - const jobId = await manager.helpers.testActiveFlow({ name: 'Ada' }) + const jobId = await manager.helpers.testActiveFlow('u/admin/live_flow', { name: 'Ada' }) expect(jobId).toBe('job-flow-preview') expect(testFlow).toHaveBeenCalledWith({ name: 'Ada' }) + // A session chat resolves an editor by its storage path, so it never names one. + expect(manager.flowAiChatHelpers).toBeUndefined() + }) + + // Session tabs keep every open flow editor mounted, so the last one to register is routinely + // a different flow than the one being tested. + it('tests the flow editor mounted on the storage path, not the last one registered', async () => { + const manager = new AIChatManager() + const testTarget = vi.fn(async () => 'job-target-flow') + const testLast = vi.fn(async () => 'job-last-flow') + + manager.setFlowHelpers( + createFlowHelpers({ testFlow: testTarget, storagePath: 'u/admin/live_flow' }) + ) + manager.setFlowHelpers( + createFlowHelpers({ testFlow: testLast, storagePath: 'u/admin/other_flow' }) + ) + + manager.changeMode(AIMode.GLOBAL) + const jobId = await manager.helpers.testActiveFlow('u/admin/live_flow', { name: 'Ada' }) + + expect(jobId).toBe('job-target-flow') + expect(testLast).not.toHaveBeenCalled() }) }) @@ -2147,6 +2408,56 @@ describe('AIChatManager queued messages', () => { ]) }) + // A checkpoint that leaves a card loading is betting the poller resolves it after + // the reload, and the poller only knows the jobs stored in the same record — + // registering one does not write it. + it('stores the job behind a card the checkpoint leaves loading', async () => { + const leavePage = stubHidingPage() + const manager = createManager() + const saveChat = vi.spyOn(manager.historyManager, 'saveChat').mockResolvedValue(undefined) + + mocks.runChatLoop.mockImplementationOnce(async (config: any) => { + config.addedMessages.push({ + role: 'assistant' as const, + content: '', + tool_calls: [ + { id: 't1', type: 'function' as const, function: { name: 'run_script', arguments: '{}' } } + ] + }) + // Inside the inline wait: the job is registered and still running, so no + // persist path has run for it yet. + manager.registerJob({ + jobId: 'job-1', + toolCallId: 't1', + kind: 'script', + label: 'f/a/b', + workspace: 'ws' + }) + config.callbacks.setToolStatus('t1', { content: 'Running...', isLoading: true }) + leavePage.forEach((fn) => fn()) + // The wait ends normally, so the only save that stored this card loading is + // the checkpoint that landed inside it. + config.callbacks.setToolStatus('t1', { content: 'Ran', isLoading: false }) + manager.updateJob('job-1', { status: 'success' }) + config.addedMessages.push({ role: 'tool' as const, tool_call_id: 't1', content: 'ran' }) + return { + addedMessages: config.addedMessages, + tokenUsage: { prompt: 0, completion: 0, total: 0 }, + hitMaxIterations: false + } + }) + + await manager.sendRequest({ instructions: 'run it' }) + + const checkpoint = saveChat.mock.calls.find(([display]) => + (display as DisplayMessage[]).some( + (m) => m.role === 'tool' && m.tool_call_id === 't1' && m.isLoading + ) + ) + expect(checkpoint).toBeDefined() + expect(checkpoint?.[4]).toEqual([expect.objectContaining({ jobId: 'job-1' })]) + }) + it('stops checkpointing once the turn commits, so the transcript is never doubled', async () => { const leavePage = stubHidingPage() const manager = createManager() @@ -2648,6 +2959,33 @@ describe('AIChatManager queued messages', () => { expect(manager.modifiedItems?.size).toBe(0) }) + // Reloading resolves no card on its own. Only the poller can, and only for the jobs + // that came back with the transcript — so a stored card without one must arrive + // settled, whichever build wrote it. + it('settles a restored loading card that no job came back to resolve', async () => { + const manager = createManager(createInputMock()) + mocks.getJob.mockResolvedValue({ type: 'QueuedJob', id: 'job-1' }) + vi.spyOn(manager.historyManager, 'loadPastChat').mockReturnValue({ + id: 'reloaded', + title: 'Reloaded', + displayMessages: [ + { role: 'tool', tool_call_id: 'orphan', content: 'Running...', isLoading: true }, + { role: 'tool', tool_call_id: 'polled', content: 'Running...', isLoading: true } + ], + actualMessages: [], + lastModified: 0 + } as unknown as ReturnType) + vi.spyOn(manager.historyManager, 'getBackgroundJobs').mockReturnValue([ + { jobId: 'job-1', toolCallId: 'polled', status: 'running' } + ] as any) + + await manager.loadPastChat('reloaded') + + const card = (id: string) => manager.displayMessages.find((m) => m.tool_call_id === id) as any + expect(card('orphan')).toMatchObject({ isLoading: false, error: 'Interrupted' }) + expect(card('polled').isLoading).toBe(true) + }) + it('seeds a session chat mask from its stored modified-items', async () => { const manager = createManager(createInputMock()) manager.isSessionChat = true @@ -3727,6 +4065,36 @@ describe('AIChatManager background job completion', () => { resultFormat: { kind: 'datatable' as const, datatableName: 'main' } } + // Live, processToolCall clears isLoading when the launching tool returns. A card + // restored from a mid-turn checkpoint never sees that return, so completing its job + // is the only thing left that can stop it spinning. + it('stops a restored card spinning when the poller completes its job', async () => { + const manager = new AIChatManager() + manager.registerJob(datatableJob) + manager.displayMessages = [ + { role: 'tool', tool_call_id: 'tc-1', content: 'Running...', isLoading: true } as any + ] + mocks.getJob.mockResolvedValue(completed({ result: [{ n: 1 }] })) + + await completeDetachedJob(manager) + + expect((manager.displayMessages[0] as any).isLoading).toBe(false) + }) + + // Streaming rides on a second endpoint; landing the job must not. A poll that always + // fails would otherwise spend the failure budget and drain a job that finished, leaving + // the card on "unreachable". + it('completes a job whose updates endpoint keeps failing', async () => { + const manager = new AIChatManager() + manager.registerJob(datatableJob) + mocks.getJobUpdates.mockRejectedValue(new Error('updates unavailable')) + mocks.getJob.mockResolvedValue(completed({ result: [{ n: 1 }] })) + + await completeDetachedJob(manager) + + expect(manager.backgroundJobs[0]?.status).toBe('success') + }) + it('reconstructs the datatable result contract from the persisted resultFormat', async () => { const manager = new AIChatManager() manager.registerJob(datatableJob) @@ -3740,12 +4108,48 @@ describe('AIChatManager background job completion', () => { // the SQL contract (row count + shaped rows) rather than generic job output. expect(applyToolStatus).toHaveBeenCalledWith('tc-1', { content: 'Query returned 2 row(s)', - result: JSON.stringify([{ n: 1 }, { n: 2 }], null, 2) + result: JSON.stringify([{ n: 1 }, { n: 2 }], null, 2), + isLoading: false }) expect(manager.pendingJobNotes).toHaveLength(1) expect(manager.pendingJobNotes[0]).toContain('"rowCount": 2') }) + // Detaching persists while the card is still loading. Storing it as interrupted would + // stick, because the patch a completed job merges in carries no error to clear. + it("stores a detached job's card unsettled, so a later success is not left an error", async () => { + const manager = new AIChatManager() + manager.registerJob(datatableJob) + manager.applyToolStatus('tc-1', { content: 'running in background', isLoading: true }) + const saveChat = vi.spyOn(manager.historyManager, 'saveChat').mockResolvedValue(undefined) + + manager.markJobDetached('job-1') + await vi.waitFor(() => expect(saveChat).toHaveBeenCalled()) + + const stored = (saveChat.mock.calls.at(-1)?.[0] as any[]).find((m) => m.tool_call_id === 'tc-1') + expect(stored.error).toBeUndefined() + expect(stored.content).toBe('running in background') + }) + + // A job still waiting inline is detached by the restore and polled like any other, so + // its card is one the poller resolves too — storing it as interrupted sticks, for the + // same reason an already-detached one would. + it("stores an inline job's card unsettled, so a later success is not left an error", async () => { + const manager = new AIChatManager() + manager.registerJob(datatableJob) + manager.registerJob({ ...datatableJob, jobId: 'job-2', toolCallId: 'tc-2' }) + manager.applyToolStatus('tc-1', { content: 'running', isLoading: true }) + const saveChat = vi.spyOn(manager.historyManager, 'saveChat').mockResolvedValue(undefined) + + // The other job reaching a terminal status is what fires the save; job-1 is still + // inside its inline wait when it lands. + manager.updateJob('job-2', { status: 'success' }) + await vi.waitFor(() => expect(saveChat).toHaveBeenCalled()) + + const stored = (saveChat.mock.calls.at(-1)?.[0] as any[]).find((m) => m.tool_call_id === 'tc-1') + expect(stored.error).toBeUndefined() + }) + it('skips reconstruction and emits no note for a canceled detached job', async () => { const manager = new AIChatManager() manager.registerJob(datatableJob) @@ -3757,9 +4161,13 @@ describe('AIChatManager background job completion', () => { // A user cancel isn't a result to shape or a completion to announce. expect(manager.pendingJobNotes).toHaveLength(0) expect(manager.backgroundJobs[0]?.status).toBe('canceled') + // The raw result, not the shaping this job's resultFormat would have applied — and no + // `error`, which is what keeps the card off the failure styling. expect(applyToolStatus).toHaveBeenCalledWith('tc-1', { content: 'Background job canceled', - logs: expect.anything() + result: expect.stringContaining('"n": 1'), + logs: expect.anything(), + isLoading: false }) }) diff --git a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte index 123768d3f3..9dd94fa4ea 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte @@ -33,6 +33,13 @@ type ReasoningProviderModel } from '../reasoningRegistry' + let { + /** Whether this dropdown carries the custom-prompt entries. Off where the surface + * has an assistant settings modal — its Instructions section owns them there, and + * two ways in would drift. The home composer has no such modal, so it keeps them. */ + promptSettings = true + }: { promptSettings?: boolean } = $props() + const aiChatManager = getAiChatManager() const AI_SETTINGS_HREF = `${base}/workspace_settings?tab=ai` @@ -335,7 +342,9 @@ class="bg-surface-tertiary dark:border w-64 origin-top-right rounded-lg shadow-lg focus:outline-none py-1 text-xs" > - + {#if promptSettings} + + {/if}
    Model
    @@ -409,21 +418,24 @@ {/snippet} - + +{#if promptSettings} + +{/if} diff --git a/frontend/src/lib/components/home/TreeView.svelte b/frontend/src/lib/components/home/TreeView.svelte index 31eae20b16..0581700316 100644 --- a/frontend/src/lib/components/home/TreeView.svelte +++ b/frontend/src/lib/components/home/TreeView.svelte @@ -32,8 +32,9 @@ // `all` pages the prefix to the end in one call instead of fetching a single page. onExpandOwner?: (prefix: string, more?: boolean, opts?: { all?: boolean }) => void onCollapseOwner?: (prefix: string) => void - // Position of this node among the rendered root nodes; "expand all" only - // auto-loads the first EXPAND_ALL_LOAD_LIMIT of them (see the effect below). + // This root owner's place in line for "expand all", which only auto-loads the first + // EXPAND_ALL_LOAD_LIMIT (see the effect below). Not always its rendered position: + // owners nested under a grouping row are ranked after the rest. rootIndex?: number showEditButton?: boolean // Path prefix of the parent node, so this one can name its own (`ownerLoad` and @@ -43,6 +44,10 @@ // is grouped under this node is only part of it: counts render as "N+" and the // node offers to load the rest of itself. ancestorHasMore?: boolean + // Visual nesting on top of `depth`. `depth` stays semantic (0 is a top-level owner + // that loads lazily), so an owner shown inside a grouping row is indented through + // this rather than by raising its depth. + indent?: number } let { @@ -59,9 +64,12 @@ rootIndex = 0, showEditButton = true, parentPrefix, - ancestorHasMore = false + ancestorHasMore = false, + indent = 0 }: Props = $props() + let visualDepth = $derived(depth + indent) + // Bounds the request burst from "expand all": however many root owners the tree // renders (its slice grows as you scroll), it fetches at most this many. Lazy owners // past the cap stay collapsed and load on a single click (see the effect). @@ -256,7 +264,7 @@ >
    0 ? `padding-left: ${depth * 16}px;` : ''} + style={visualDepth > 0 ? `padding-left: ${visualDepth * 16}px;` : ''} >
    {#if isUser(item)} @@ -310,7 +318,7 @@ Pipeline @@ -335,12 +343,13 @@ {showCode} {showEditButton} depth={depth + 1} + {indent} /> {/each} {#if effectiveMax < item.items.length}
    @@ -377,7 +386,7 @@ as rows still missing. -->
    Showing {loadedHere}{ownerTotal != undefined ? ` of ${ownerTotal}` : ''} items in {nodePrefix} @@ -427,6 +436,6 @@ on:appChanged on:rawAppChanged on:reload - {depth} + depth={visualDepth} /> {/if} diff --git a/frontend/src/lib/components/home/TreeViewRoot.svelte b/frontend/src/lib/components/home/TreeViewRoot.svelte index 39c2188e59..062dfb69f8 100644 --- a/frontend/src/lib/components/home/TreeViewRoot.svelte +++ b/frontend/src/lib/components/home/TreeViewRoot.svelte @@ -1,8 +1,10 @@ +{#snippet ownerNode(node: RootNode, loadRank: number, indent: number)} + +{/snippet} + {#if groupedItems === 'loading'}
    {:else}
    - {#each groupedItems.slice(0, nbDisplayed) as item, rootIndex ('folderName' in item ? `f__${item.folderName}` : 'username' in item ? `u__${item.username}` : `i__${item.type}__${item.path}`)} - {#if item} - + {#each rows.slice(0, nbDisplayed) as row (row.kind === 'otherUsers' ? 'other_users' : 'folderName' in row.node ? `f__${row.node.folderName}` : 'username' in row.node ? `u__${row.node.username}` : `i__${row.node.type}__${row.node.path}`)} + {#if row.kind === 'otherUsers'} + + + +
    +
    + +
    + Other users +
    + ({pluralize(otherUsers.length, 'user')}{otherUsersItemCount != undefined + ? ` · ${pluralize(otherUsersItemCount, 'item')}` + : ''}) +
    +
    +
    +
    + {#if otherUsersOpen} + + {#each otherUsers.slice(0, nbOtherUsersDisplayed) as user, i (user.username)} + {@render ownerNode(user, ownerRowCount + i, 1)} + {/each} + {#if nbOtherUsersDisplayed < otherUsers.length} +
    + + Showing {nbOtherUsersDisplayed} of {otherUsers.length} users + + +
    + {/if} + {/if} + {:else} + {@render ownerNode(row.node, row.loadRank, 0)} {/if} {/each} - {#if nbDisplayed < groupedItems.length || hasMoreServer} + {#if nbDisplayed < rows.length || hasMoreServer}
    - {#if nbDisplayed < groupedItems.length} - Showing {nbDisplayed} of {groupedItems.length} folders and users + {#if nbDisplayed < rows.length} + Showing {shownOwnerRowCount} of {ownerRowCount} folders and users {:else} @@ -231,12 +355,12 @@ unifiedSize="sm" variant="subtle" on:click={() => { - if (nbDisplayed < groupedItems.length) - nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, groupedItems.length) + if (nbDisplayed < rows.length) + nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, rows.length) else onLoadMore?.() }} > - {nbDisplayed < groupedItems.length ? 'Show more' : 'Load more'} + {nbDisplayed < rows.length ? 'Show more' : 'Load more'}
    {/if} diff --git a/frontend/src/lib/components/home/TutorialBanner.svelte b/frontend/src/lib/components/home/TutorialBanner.svelte deleted file mode 100644 index ef7b30c27e..0000000000 --- a/frontend/src/lib/components/home/TutorialBanner.svelte +++ /dev/null @@ -1,178 +0,0 @@ - - -{#if !isDismissed} - -
    - - {#if hasCompletedAny} - New tutorial available! - {:else} - First time? - {/if} - - - -
    -{/if} diff --git a/frontend/src/lib/components/home/TutorialButton.svelte b/frontend/src/lib/components/home/TutorialButton.svelte deleted file mode 100644 index 6a31ae9569..0000000000 --- a/frontend/src/lib/components/home/TutorialButton.svelte +++ /dev/null @@ -1,124 +0,0 @@ - - - - diff --git a/frontend/src/lib/components/home/WorkspaceEmptyState.svelte b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte new file mode 100644 index 0000000000..fc36cc334b --- /dev/null +++ b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte @@ -0,0 +1,146 @@ + + +
    + {#each rowOpacities as opacity, i (i)} + + {/each} + + +
    + {#if archivedOnly} + + + Everything in this workspace is archived. + . + + {:else} + Your scripts, flows and apps will show up here. + {/if} + {#if canCreate} + + {#if !$disableHubStore} + + + e.detail && logFeatureUsage('home', 'template_picker_open', { key: 'empty_state' })} + > + {#snippet trigger()}Start from a template{/snippet} + {#snippet content({ close })} + { + close() + onPick(project) + }} + /> + {/snippet} + + or + {/if} + + {#snippet trigger()} + + . + {/snippet} + + {/if} +
    +
    diff --git a/frontend/src/lib/components/icons/BRAND_COLORS.md b/frontend/src/lib/components/icons/BRAND_COLORS.md index 1cff77a3d3..11d4df5c3b 100644 --- a/frontend/src/lib/components/icons/BRAND_COLORS.md +++ b/frontend/src/lib/components/icons/BRAND_COLORS.md @@ -179,6 +179,7 @@ repeatedly — check the brand's own page. | `KeycloakIcon` | `keycloak` | fixed | #00B8E3 | #00B8E3 | 8.18 | 10.65 | keycloak.org's own mark, https://www.keycloak.org/resources/images/icon.svg (cyan #00B8E3/#33C6E9/#008AAA over greys #4D4D4D–#EDEDED, single theme) | | `KlaviyoIcon` | `klaviyo` | pair | #1D1E20 | #FFFFFF | 16.14 | 12.47 | klaviyo.com --color-core-charcoal; the flag mark is the standalone logomark the site header collapses to, and the shape of klaviyo.com/icons/icon-512x512.png | | `KoboToolboxIcon` | `kobotoolbox` | fixed | #2095F3 | #2095F3 | 3.05 | 3.95 | the kobotoolbox.org header logo and $kobo-blue in kobotoolbox/kpi jsapp/scss/colors.scss | +| `KubernetesIcon` | — | fixed | #326CE5 | #326CE5 | 4.61 | 2.62 | the CNCF artwork repo (github.com/cncf/artwork/projects/kubernetes/icon/color/kubernetes-icon-color.svg, CC-BY-4.0) | | `KustomerIcon` | `kustomer` | fixed | #FBEC2A | #FBEC2A | 14.08 | 12.47 | kustomer.com/images/kustomer/Kusty.svg | | `LangfuseIcon` | `langfuse` | fixed | #FF5D5F | #FF5D5F | 2.91 | 4.47 | langfuse.com/brand "Icon - Color (SVG)", used unmodified | | `LessIcon` | — | pair | #274F82 | #FFFFFF | 8.04 | 12.47 | github.com/less/logo (MIT) | @@ -443,6 +444,7 @@ Constraints that would otherwise be broken by a well-meaning change. - **JsonIcon** — JSON itself has no brand owner or published colours — json.org states none — so this is a Material palette pick, not a brand colour. - **KanidmIcon** — Kanidm's artwork is CC-BY-NC-ND — no recolouring or other derivatives. - **KlaviyoIcon** — Klaviyo draws it in currentColor, hence the white swap on dark. +- **KubernetesIcon** — The Linux Foundation trademark guidelines allow only the published variants (colour, all-blue, black, white); the colour mark's white knockouts are part of the artwork, not a theme swap. - **LangfuseIcon** — Langfuse's trademark terms forbid modifying the assets. - **LineIcon** — LINE forbids any change to the logo's colour, so there is no reversed variant. - **LinearIcon** — Guidelines ship a light/dark logomark pair and forbid altering the assets in any other way. @@ -571,7 +573,7 @@ Not brands. These inherit `currentColor` on purpose and must not be given a pair ## Coverage -- brand icons: **314**, of which **310** carry a recorded source +- brand icons: **315**, of which **311** carry a recorded source - per-theme pairs applied: **136** (5 of them by inversion or a two-SVG swap, see above) - concept icons: **34** - effectively invisible on light: **1** (AbstractApiIcon) diff --git a/frontend/src/lib/components/icons/KubernetesIcon.svelte b/frontend/src/lib/components/icons/KubernetesIcon.svelte new file mode 100644 index 0000000000..76279e002b --- /dev/null +++ b/frontend/src/lib/components/icons/KubernetesIcon.svelte @@ -0,0 +1,35 @@ + + + + + + + + + diff --git a/frontend/src/lib/components/inputTransformEnv.svelte.ts b/frontend/src/lib/components/inputTransformEnv.svelte.ts new file mode 100644 index 0000000000..50df45a15a --- /dev/null +++ b/frontend/src/lib/components/inputTransformEnv.svelte.ts @@ -0,0 +1,41 @@ +import { CancelError, WorkspaceService } from '$lib/gen' +import { resource } from 'runed' + +/** + * Whether the workspace has S3 storage configured, for the fields that warn without it. Call during + * component initialisation and read `.current` where the answer is used. + */ +export function useS3StorageConfigured(ws: () => string | undefined): { + readonly current: boolean +} { + const settings = resource(ws, async (ws, _previousWs, { onCleanup }) => { + if (!ws) return undefined + const req = WorkspaceService.getPublicSettings({ workspace: ws }) + // `resource` keeps whatever lands last: cancel a superseded request so a slow + // reply for a workspace we have left cannot overwrite the current one. + onCleanup(() => req.cancel()) + try { + return { ws, settings: await req } + } catch (err) { + if (!(err instanceof CancelError)) { + console.error('Failed to fetch workspace settings:', err) + } + return undefined + } + }) + + // Assume configured until this workspace's own answer lands: the warning must not + // linger from the previous workspace, nor appear merely because the fetch failed. + const configured = $derived.by(() => { + const loaded = settings.current + return loaded && loaded.ws === ws() + ? loaded.settings.large_file_storage?.s3_resource_path !== undefined + : true + }) + + return { + get current() { + return configured + } + } +} diff --git a/frontend/src/lib/components/instanceBanner.test.ts b/frontend/src/lib/components/instanceBanner.test.ts new file mode 100644 index 0000000000..bc93af18a1 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from 'vitest' +import { + INSTANCE_BANNER_MESSAGE_MAX_LEN, + isInstanceBannerVisible, + resolveInstanceBanner +} from './instanceBanner' + +const BANNER = { + enabled: true, + message: 'Scheduled maintenance on Saturday.', + severity: 'warning', + dismissible: true +} + +describe('resolveInstanceBanner', () => { + it('drops a link that is not an absolute http(s) URL', () => { + // Declarative instance config writes global_settings rows directly, so the API's + // scheme check is not the only thing standing between a stored value and an href. + for (const link of ['javascript:alert(1)', 'data:text/html,x', 'status.example.com', 123]) { + expect(resolveInstanceBanner({ ...BANNER, link })?.link).toBeUndefined() + } + expect(resolveInstanceBanner({ ...BANNER, link: 'https://status.example.com' })?.link).toBe( + 'https://status.example.com' + ) + }) + + it('keeps a message the backend accepted whole', () => { + // The backend caps at INSTANCE_BANNER_MESSAGE_MAX_LEN code points (`chars().count()`). + // Truncating with `slice` here would count UTF-16 units and halve an all-emoji message + // that passed validation, so the two sides must measure the same way. + const emoji = '\u{1F6A7}'.repeat(INSTANCE_BANNER_MESSAGE_MAX_LEN) + expect([...resolveInstanceBanner({ ...BANNER, message: emoji })!.message]).toHaveLength( + INSTANCE_BANNER_MESSAGE_MAX_LEN + ) + }) + + it('shows nothing when disabled or without a message', () => { + // An enabled banner with no message is a writable state (the backend accepts it so a + // half-typed announcement cannot fail an admin's whole settings save), so this is the + // only thing keeping it off everyone's screen. + expect(resolveInstanceBanner({ ...BANNER, enabled: false })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: ' ' })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: 42 })).toBeUndefined() + }) +}) + +describe('isInstanceBannerVisible', () => { + it('honours a dismissal only while the announcement is dismissible', () => { + const dismissible = resolveInstanceBanner(BANNER)! + expect(isInstanceBannerVisible(dismissible, dismissible.fingerprint)).toBe(false) + expect(isInstanceBannerVisible(dismissible, 'some other announcement')).toBe(true) + + // Escalating the same announcement to mandatory must reach the people who already + // dismissed it — the fingerprint does not change, so nothing else would bring it back. + const mandatory = resolveInstanceBanner({ ...BANNER, dismissible: false })! + expect(mandatory.fingerprint).toBe(dismissible.fingerprint) + expect(isInstanceBannerVisible(mandatory, mandatory.fingerprint)).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/instanceBanner.ts b/frontend/src/lib/components/instanceBanner.ts new file mode 100644 index 0000000000..f1693024b8 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.ts @@ -0,0 +1,129 @@ +import { SettingService } from '$lib/gen' + +/** Drives the banner palette and icon; the subset of `AlertType` that fits an announcement. */ +export type InstanceBannerSeverity = 'info' | 'warning' | 'error' + +/** Stored shape of the `instance_banner` global setting. Every field is optional: a + * stored value can predate a field this code knows about. */ +export interface InstanceBanner { + enabled?: boolean + message?: string + severity?: InstanceBannerSeverity + /** Whether a viewer may dismiss the banner for themselves. Absent means yes. */ + dismissible?: boolean + link?: string + link_label?: string +} + +export const INSTANCE_BANNER_SETTING = 'instance_banner' + +/** Mirror `INSTANCE_BANNER_MESSAGE_MAX_LEN` / `INSTANCE_BANNER_LINK_LABEL_MAX_LEN` in + * backend/windmill-common/src/global_settings.rs, which reject longer values at write time. */ +export const INSTANCE_BANNER_MESSAGE_MAX_LEN = 500 +export const INSTANCE_BANNER_LINK_LABEL_MAX_LEN = 60 + +export type ResolvedInstanceBanner = { + message: string + severity: InstanceBannerSeverity + dismissible: boolean + link?: string + linkLabel: string + /** Dismissal token: a viewer who dismissed one announcement sees the next one, + * because editing any displayed part of the banner changes this string. */ + fingerprint: string +} + +/** + * Read a stored banner field as a string. + * + * The setting is a raw `global_settings` row, so its shape is only ever as good as the + * writer that last touched it — and anything here that calls `.trim()` on a number throws, + * taking the whole settings form down with it. + */ +export function bannerString(value: unknown): string { + return typeof value === 'string' ? value : '' +} + +/** + * Truncate to `max` code points, matching the backend's `chars().count()` cap. + * + * `String.slice` counts UTF-16 code units, so it would cut a 500-emoji message the backend + * accepted in half — and the two sides must agree on what "500 characters" means. + */ +function truncateChars(value: string, max: number): string { + const chars = [...value] + return chars.length > max ? chars.slice(0, max).join('') : value +} + +export function isHttpUrl(value: string): boolean { + try { + const url = new URL(value) + return url.protocol === 'http:' || url.protocol === 'https:' + } catch { + return false + } +} + +/** + * Turn the raw setting into what the banner renders, or `undefined` for "show nothing". + * + * The scheme check repeats the one the writers run on purpose. The link becomes the href of + * an anchor shown to every user of the instance, and this is the last place that can refuse + * it — a row predating the validator, or written straight to the table, reaches here having + * passed nothing. + */ +export function resolveInstanceBanner(raw: unknown): ResolvedInstanceBanner | undefined { + if (!raw || typeof raw !== 'object') return undefined + const banner = raw as InstanceBanner + const message = bannerString(banner.message).trim() + if (banner.enabled !== true || message === '') return undefined + + const severity: InstanceBannerSeverity = + banner.severity === 'warning' || banner.severity === 'error' ? banner.severity : 'info' + const rawLink = bannerString(banner.link).trim() + const link = isHttpUrl(rawLink) ? rawLink : undefined + const linkLabel = + truncateChars(bannerString(banner.link_label).trim(), INSTANCE_BANNER_LINK_LABEL_MAX_LEN) || + 'Learn more' + + return { + message: truncateChars(message, INSTANCE_BANNER_MESSAGE_MAX_LEN), + severity, + dismissible: banner.dismissible !== false, + link, + linkLabel, + fingerprint: JSON.stringify([message, severity, link ?? '', link ? linkLabel : '']) + } +} + +/** + * The reason the banner form cannot be saved, or `undefined` when it can. + * + * Shared with the setting's `isValid` so the Save button and the inline message agree: the + * backend refuses a bad link, and a category save fires its settings concurrently, so a Save + * that got this far would persist the other Core settings and fail only the banner. + */ +export function instanceBannerFormError(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const link = bannerString((value as InstanceBanner).link).trim() + return link !== '' && !isHttpUrl(link) ? 'Link must be an absolute http(s) URL' : undefined +} + +/** + * Whether a viewer holding `dismissedFingerprint` should see this announcement. + * + * A non-dismissible announcement ignores stored dismissals entirely: an admin escalating an + * existing notice to mandatory must reach the people who already dismissed it, and the + * fingerprint deliberately does not cover `dismissible`, so nothing else would bring it back. + */ +export function isInstanceBannerVisible( + banner: ResolvedInstanceBanner | undefined, + dismissedFingerprint: string +): banner is ResolvedInstanceBanner { + if (banner == undefined) return false + return !banner.dismissible || dismissedFingerprint !== banner.fingerprint +} + +export async function fetchInstanceBanner(): Promise { + return resolveInstanceBanner(await SettingService.getGlobal({ key: INSTANCE_BANNER_SETTING })) +} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index eb8c4c86c4..0677648978 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -1,5 +1,7 @@ import type { ButtonType } from './common/button/model' +import { allowedOriginsSettingError } from './triggers/http/utils' import { z } from 'zod' +import { instanceBannerFormError } from './instanceBanner' import { writable } from 'svelte/store' /** @@ -68,6 +70,7 @@ export interface Setting { | 'webhook_base_url' | 'ws_connectivity' | 'retention_overrides' + | 'instance_banner' storage: SettingStorage advancedToggle?: { label: string @@ -236,6 +239,19 @@ export const settings: Record = { placeholder: 'only for EE', storage: 'setting' }, + { + label: 'Announcement banner', + description: + 'Message shown above every page of the instance, for maintenance windows and incidents.', + key: 'instance_banner', + fieldType: 'instance_banner', + storage: 'setting', + // The banner only renders on the managed cloud, so only offer it there. + cloudonly: true, + hideInQuickSetup: true, + // Gates Save. The card renders the specific message itself, so no `error` here. + isValid: (value: any) => instanceBannerFormError(value) == undefined + }, { label: 'Non-prod instance', description: @@ -266,6 +282,22 @@ export const settings: Record = { ee_only: '', hideInQuickSetup: true }, + { + label: 'HTTP route default allowed origins', + description: + 'Origins that HTTP routes allow to call them from a browser when the route sets none of its own. A route overrides this with its own list, and opts out entirely by setting its allowed origins to *. Leave unset for no instance-wide default, so every route is callable from any origin unless it restricts itself.', + key: 'http_route_default_allowed_origins', + fieldType: 'text', + placeholder: 'https://app.example.com, https://admin.example.com', + storage: 'setting', + error: + 'Each origin must be visible ASCII with no comma, and there can be at most 100 of them. null is not allowed, since every sandboxed iframe sends it.', + // The same check the API applies, so a value it would refuse cannot be + // saved here and then silently drop to no restriction at the next boot. + isValid: (value: unknown) => allowedOriginsSettingError(value) === undefined, + ee_only: '', + hideInQuickSetup: true + }, { label: 'Audit log retention (days)', key: 'audit_log_retention_days', @@ -662,6 +694,16 @@ export const settings: Record = { fieldType: 'text', placeholder: 'okta', storage: 'setting' + }, + { + label: 'SSO groups claim', + description: + 'Name of the SAML attribute or OIDC userinfo claim carrying the user\'s IdP groups ("http://schemas.microsoft.com/ws/2008/06/identity/claims/groups" on Entra SAML, "groups" for most OIDC providers). Its values must be the same group ids that SCIM stored as the instance groups\' external id (Entra emits object ids in both), since matching is by external id only. When set, every SSO login reconciles the user\'s membership in those SCIM-provisioned instance groups against the claim, so IdP group changes take effect at the next login instead of waiting for the SCIM push. Instance groups without an external id are never touched, and a login whose claim is absent or empty changes nothing. Leave empty to disable.', + key: 'sso_groups_claim', + fieldType: 'text', + placeholder: 'groups', + storage: 'setting', + ee_only: '' } ], 'DB Health': [], diff --git a/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte new file mode 100644 index 0000000000..d55c2f9ce6 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte @@ -0,0 +1,133 @@ + + + + +{#if !usage.available} +
    + + No guest can sign in here, whatever a workspace or an app says. Guests require a self-hosted + instance or a dedicated Windmill Cloud deployment. + +
    +{:else} +
    + {#key usage} + setInstanceSwitch(e.detail)} + options={{ + right: 'Allow guests on this instance', + rightTooltip: + 'Off, no guest can sign in anywhere, whatever a workspace or an app says, and sessions already issued stop on their next request.' + }} + /> + {/key} +
    + +
    + + {#if usage.metered} + Beyond the allowance, every four guests count as one seat{usage.guest_seats > 0 + ? `: ${usage.billable_guests} guests past it take ${usage.guest_seats} ${usage.guest_seats === 1 ? 'seat' : 'seats'} now` + : ''}. + {:else} + Beyond the allowance, new guests are refused until the count drops below it; an Enterprise + license meters them instead. + {/if} + +
    +{/if} + + onLoadMore()} +> + +
    + Email + Workspaces + First seen + Last seen + + + + {#each guests as guest, i (guest.email)} + + {guest.email} + {guest.workspaces.join(', ')} + {guest.first_seen} + {guest.last_seen} + + {/each} + + diff --git a/frontend/src/lib/components/instanceSettings/InstanceBannerSetting.svelte b/frontend/src/lib/components/instanceSettings/InstanceBannerSetting.svelte new file mode 100644 index 0000000000..293f567a0b --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/InstanceBannerSetting.svelte @@ -0,0 +1,159 @@ + + +
    + banner.enabled === true, (v) => ($values[INSTANCE_BANNER_SETTING].enabled = v) + } + options={{ right: 'Show the banner to every user' }} + /> + +
    + Message + bannerString(banner.message), + (v) => ($values[INSTANCE_BANNER_SETTING].message = String(v)) + } + /> +
    + +
    + Severity + bannerString(banner.severity) || 'info', + (v) => ($values[INSTANCE_BANNER_SETTING].severity = v) + } + > + {#snippet children({ item })} + {#each severities as severity (severity.value)} + + {/each} + {/snippet} + +
    + +
    + Link (optional) +
    + bannerString(banner.link), + (v) => ($values[INSTANCE_BANNER_SETTING].link = String(v)) + } + /> + bannerString(banner.link_label), + (v) => ($values[INSTANCE_BANNER_SETTING].link_label = String(v)) + } + /> +
    + {#if linkError} + {linkError} + {/if} +
    + + banner.dismissible !== false, (v) => ($values[INSTANCE_BANNER_SETTING].dismissible = v) + } + options={{ + right: 'Let users dismiss it', + rightTooltip: + 'Dismissal is remembered per browser and only for this exact announcement: editing the message, severity or link brings it back for everyone. Turning this off also shows it again to everyone who had dismissed it.' + }} + /> + +
    + Preview + {#if preview} + {@const palette = alertClasses[preview.severity]} + {@const Icon = alertIcons[preview.severity]} +
    + + {preview.message} + {#if preview.link} + {preview.linkLabel} + {/if} +
    + {:else} + + {banner.enabled === true + ? 'Nothing is shown until the message is filled in.' + : 'Nothing is shown while the banner is off.'} + + {/if} +
    +
    diff --git a/frontend/src/lib/components/job_args.test.ts b/frontend/src/lib/components/job_args.test.ts new file mode 100644 index 0000000000..cf5818ff0f --- /dev/null +++ b/frontend/src/lib/components/job_args.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from 'vitest' +import { + coerceArgsToSchema, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs +} from './job_args' + +describe('coerceArgsToSchema', () => { + // A scalar widget renders its own reading of a wrong-typed value and never writes that + // reading back, so an untouched form submits something it never displayed: a number + // input paints `"12"` as a filled-looking 12, and a toggle shows `"false"` as on. + it('converts a value its widget would read, so the form shows what runs', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' }, + name: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: '12', flag: 'false', label: 3, name: 'ada' }, + schema + ) + expect(args).toEqual({ count: 12, flag: false, label: '3', name: 'ada' }) + expect(clearedKeys).toEqual([]) + }) + + // Cleared, not carried: the widget shows nothing for these, so nothing is what an + // untouched form should send. + it('empties a value with no reading in its declared type', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: 'abc', flag: 'maybe', label: { a: 1 } }, + schema + ) + expect(args).toEqual({}) + expect(clearedKeys.sort()).toEqual(['count', 'flag', 'label']) + }) + + // A field the schema does not name is a field no run surface in the product draws, so a + // value under that name would reach the job without anyone having been able to see it. + // `constructor` is declared by every object through its prototype and by no schema. + it('drops arguments the schema does not declare, naming them', () => { + const kept = coerceArgsToSchema({ a: 'keep', b: 2, constructor: 'x' }, { + properties: { a: { type: 'string' } } + } as any) + expect(kept.args).toEqual({ a: 'keep' }) + expect(kept.undeclaredKeys).toEqual(['b', 'constructor']) + // Declaring nothing is declaring no arguments, which is what a `**kwargs` script and a + // schema that failed to infer both look like. + expect(coerceArgsToSchema({ a: 1 }, undefined).args).toEqual({}) + }) + + // Resolved by the job, so the declared type describes what it receives and never the + // string standing in for it. `Number('$var:…')` is NaN, so coercing would destroy it. + it('leaves a variable or resource reference in any slot', () => { + const schema = { + properties: { + size: { type: 'number' }, + on: { type: 'boolean' }, + db: { type: 'object', format: 'resource-postgresql' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { size: '$var:u/admin/size', on: '$var:u/admin/on', db: '$res:u/admin/pg' }, + schema + ) + expect(args).toEqual({ + size: '$var:u/admin/size', + on: '$var:u/admin/on', + db: '$res:u/admin/pg' + }) + expect(clearedKeys).toEqual([]) + }) + + // Not merely unreadable: `MultiSelect` maps over the value as it renders, so anything + // else throws and takes the whole card down, Cancel with it. A reference is no + // exception — the widget draws before anything resolves — so this slot is the one + // place the reference rule above does not hold. + it('empties a non-array in a dyn-multiselect slot, reference included', () => { + const schema = { properties: { tags: { type: 'object', format: 'dynmultiselect-list' } } } + expect(coerceArgsToSchema({ tags: ['a'] }, schema).args).toEqual({ tags: ['a'] }) + for (const bad of [{ a: 1 }, '$var:u/admin/watchlist']) { + const { args, clearedKeys } = coerceArgsToSchema({ tags: bad }, schema) + expect(args).toEqual({}) + expect(clearedKeys).toEqual(['tags']) + } + }) + + // Below the top the form has the same limitations as everywhere else in the product, + // and descending means resolving `oneOf` branches — where being wrong rewrites what the + // user typed into the branch they did open. + it('leaves nested and container values to the widget that renders them', () => { + const schema = { + properties: { + obj: { type: 'object', properties: { known: { type: 'string' } } }, + rows: { type: 'array', items: { type: 'object' } } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }, + schema + ) + expect(args).toEqual({ obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }) + expect(clearedKeys).toEqual([]) + }) + + // Both sides parsed, never written as literals: `__proto__:` in an object literal is + // the prototype setter, so a literal declares nothing to coerce in the first place. + it('keeps a declared __proto__ instead of losing it to the setter', () => { + const { args } = coerceArgsToSchema( + JSON.parse('{"__proto__":"legit","keep":1}'), + JSON.parse('{"properties":{"__proto__":{"type":"string"},"keep":{"type":"number"}}}') + ) + expect(Object.hasOwn(args, '__proto__')).toBe(true) + expect(args['__proto__']).toBe('legit') + }) +}) + +describe('enforceDisabledDefaults', () => { + const schema = { + properties: { + locked: { type: 'string', disabled: true, default: 'fixed' }, + open: { type: 'string' } + } + } + + it('overwrites a disabled field and reports only what it changed', () => { + expect(enforceDisabledDefaults({ locked: 'mine', open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: ['locked'] + }) + // Never supplied is not overwritten: the field shows the default either way, and a + // caller told otherwise would try to correct what it never sent. + expect(enforceDisabledDefaults({ open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: [] + }) + }) + + it('reports no reset for an object default the caller already matched', () => { + const objSchema = { + properties: { conf: { type: 'object', disabled: true, default: { a: 1 } } } + } + expect(enforceDisabledDefaults({ conf: { a: 1 } }, objSchema).resetKeys).toEqual([]) + }) +}) + +describe('secret args at every level the form nests', () => { + const schema = { + properties: { + top: { type: 'string', password: true }, + obj: { properties: { inner: { type: 'string', password: true } } }, + list: { items: { properties: { secret: { type: 'string', password: true } } } }, + either: { + oneOf: [ + { title: 'a', properties: { key: { type: 'string', password: true } } }, + { title: 'b', properties: { other: { type: 'string', password: true } } } + ] + } + } + } + const args = { + top: 'hunter2', + obj: { inner: '$var:u/ada/prod', keep: 1 }, + list: [{ secret: 'one', name: 'a' }, { secret: 'two' }], + // Tagged as branch 'a', but 'b' is stripped too: the tag is runtime state. + either: { kind: 'a', key: 'k', other: 'o' } + } + + it('redacts every value and keeps every reference', () => { + const redacted = JSON.stringify(redactSecretArgs(args, schema)) + for (const secret of ['hunter2', 'one', 'two', '"k"', '"o"']) { + expect(redacted).not.toContain(secret) + } + expect(redacted).toContain('') + expect(redacted).toContain('"name":"a"') + expect(redacted).toContain('$var:u/ada/prod') + }) + + // ArgInput synthesises '' for every untouched string, so marking one would put a hidden + // value on the card for a field nobody filled in — and mint nothing to back it. + it('leaves an empty secret empty', () => { + expect( + redactSecretArgs({ tok: '' }, { properties: { tok: { type: 'string', password: true } } }) + ).toEqual({ tok: '' }) + }) + + it('reaches a secret under a oneOf branch of an array element', () => { + const oneOfItems = { + properties: { + steps: { + type: 'array', + items: { + oneOf: [{ title: 'push', properties: { token: { type: 'string', password: true } } }] + } + } + } + } + expect(redactSecretArgs({ steps: [{ token: 'hunter2', name: 'a' }] }, oneOfItems)).toEqual({ + steps: [{ token: '', name: 'a' }] + }) + }) + + // The walk descends on the value's shape: routing an array down `properties` because the + // declaration carries that key would visit none of its elements, leaving the secret in + // the persisted card verbatim. + it('reaches through a declaration carrying both items and properties', () => { + const both = { + properties: { + creds: { + type: 'array', + items: { properties: { token: { type: 'string', password: true } } }, + properties: { token: { type: 'string', password: true } } + } + } + } + expect(redactSecretArgs({ creds: [{ token: 'hunter2' }] }, both)).toEqual({ + creds: [{ token: '' }] + }) + }) + + // A container shaped unlike its declaration is kept, so the walk has to reach in through + // the half the declaration does carry — on the value's shape alone it stops at the + // mismatch, leaving the secret there for the persisted card and the model to read. + it('reaches through a container shaped unlike its declaration', () => { + const declaresArray = { + properties: { + rows: { type: 'array', items: { properties: { token: { password: true } } } } + } + } + expect(redactSecretArgs({ rows: { token: 'hunter2' } }, declaresArray)).toEqual({ + rows: { token: '' } + }) + + const declaresObject = { + properties: { cfg: { type: 'object', properties: { token: { password: true } } } } + } + expect(redactSecretArgs({ cfg: [{ token: 'hunter2' }] }, declaresObject)).toEqual({ + cfg: [{ token: '' }] + }) + }) +}) + +describe('redactFileArgs', () => { + const schema = { + properties: { + doc: { type: 'string', contentEncoding: 'base64' }, + pics: { type: 'array', items: { type: 'string', contentEncoding: 'base64' } }, + wrap: { properties: { inner: { type: 'string', contentEncoding: 'base64' } } }, + note: { type: 'string' } + } + } + + it('replaces the bytes with a size marker at every level, and keeps the rest', () => { + const oneMeg = 'A'.repeat(1024 * 1024 * 2) + const redacted = redactFileArgs( + { doc: oneMeg, pics: ['B'.repeat(4096)], wrap: { inner: 'C'.repeat(2048) }, note: 'hi' }, + schema + ) + expect(redacted).toEqual({ + doc: '', + pics: [''], + wrap: { inner: '' }, + note: 'hi' + }) + }) +}) diff --git a/frontend/src/lib/components/job_args.ts b/frontend/src/lib/components/job_args.ts index ef3377081f..9698c03cd4 100644 --- a/frontend/src/lib/components/job_args.ts +++ b/frontend/src/lib/components/job_args.ts @@ -1,5 +1,303 @@ +/** + * A job's arguments prepared for a run form, its readers, and a result view. Coercing must + * not lose what the caller meant to send, so it is exact and shallow; stripping and + * redacting only blank a field, so they go to any depth and err towards visiting too much. + */ import { deepEqual } from 'fast-equals' +const isLockedProp = (prop: any) => !!prop?.disabled && 'default' in prop + +/** + * A field the schema disables is not the caller's to set: the run sends the schema's + * default whatever it holds. Top-level only, like every filter here. Returns the keys it + * overwrote; notifying is the caller's job. + */ +export function enforceDisabledDefaults( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; resetKeys: string[] } { + // Null prototype: assigning a declared `__proto__` into a plain `{}` reaches the + // inherited setter and the default vanishes. Always copied — callers bind the result to + // a form that edits in place, so returning the input would write through to theirs. + const result: Record = Object.assign(Object.create(null), args) + if (!schema?.properties) return { args: { ...result }, resetKeys: [] } + const resetKeys: string[] = [] + for (const [key, prop] of Object.entries(schema.properties)) { + if (!isLockedProp(prop)) continue + // Never supplied is not overwritten, and compared by value: a default can be an + // object, where identity would report every correct run as overridden. + if (result[key] !== undefined && !deepEqual(result[key], prop.default)) resetKeys.push(key) + result[key] = prop.default + } + return { args: { ...result }, resetKeys } +} + +/** How a form says what {@link enforceDisabledDefaults} overwrote, shared by the two that + * run it so the wording cannot drift apart. */ +export const resetKeysToast = (resetKeys: string[]): string => + `Disabled field${resetKeys.length > 1 ? 's' : ''} ${resetKeys + .map((k) => `'${k}'`) + .join(', ')} reset to default value${resetKeys.length > 1 ? 's' : ''}` + +/** Types `setInputCat` routes to a widget bound to a scalar. */ +const SCALAR_TYPES = new Set(['string', 'number', 'integer', 'boolean']) + +/** + * Declares an array though its `type` says `object`. A mismatch here throws rather than + * reading wrong: `MultiSelect` maps over the value as it renders, so anything else takes + * the form down, Cancel with it — a reference included, since it draws before resolving. + */ +const declaresDynMultiselect = (prop: any) => + typeof prop?.format === 'string' && prop.format.startsWith('dynmultiselect-') + +const fitsScalarType = (value: any, type: string): boolean => + type === 'integer' ? typeof value === 'number' : typeof value === type + +/** + * Resolved at run time, so the declared type describes what the job receives and never the + * string standing in for it. `ArgInput.validateInput` blesses these ahead of every type check. + */ +const REFERENCE_PREFIXES = ['$var:', '$res:', '$jsonvar:'] +const isReference = (value: any): boolean => + typeof value === 'string' && REFERENCE_PREFIXES.some((prefix) => value.startsWith(prefix)) + +/** No plain reading in the declared type; distinct from a value that reads as `undefined`. */ +const UNCOERCIBLE = Symbol('uncoercible') + +/** + * The value a scalar widget would stand for, or {@link UNCOERCIBLE}. Only conversions with + * one plain reading: a number input shows `"7"` as 7 and a toggle shows any non-empty + * string as on, so guessing past this would put a value on screen that nobody wrote. + */ +function coerceScalar(value: any, type: string): any { + if (typeof value === 'object') return UNCOERCIBLE + if (type === 'string') { + return typeof value === 'number' || typeof value === 'boolean' ? String(value) : UNCOERCIBLE + } + if (typeof value !== 'string') return UNCOERCIBLE + const trimmed = value.trim() + if (type === 'number' || type === 'integer') { + if (trimmed === '') return UNCOERCIBLE + const parsed = Number(trimmed) + return Number.isFinite(parsed) ? parsed : UNCOERCIBLE + } + if (type === 'boolean') { + if (trimmed.toLowerCase() === 'true') return true + if (trimmed.toLowerCase() === 'false') return false + } + return UNCOERCIBLE +} + +/** + * Drop every argument the schema does not declare, naming them. The schema is what every run + * surface builds its fields from, so a value under a name it never declares has no widget + * anywhere: sending one is sending what nobody could see or edit before the run. + */ +export function dropUndeclaredArgs( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; undeclaredKeys: string[] } { + const properties = schema?.properties ?? {} + // hasOwn, not `in`: every object inherits `constructor` and `toString`, so `in` would + // hand an inherited declaration to an argument the schema never named. + const kept: Record = Object.create(null) + const undeclaredKeys: string[] = [] + for (const [key, value] of Object.entries(args ?? {})) { + if (Object.hasOwn(properties, key)) kept[key] = value + else undeclaredKeys.push(key) + } + return { args: { ...kept }, undeclaredKeys } +} + +/** + * Make arguments say what the run form will show, then apply {@link enforceDisabledDefaults}. + * A scalar widget renders its own reading of a wrong-typed value and never writes it back, so + * an untouched form would submit what it never displayed; a value with no reading is cleared. + * Top-level only: descending means resolving `oneOf`, where being wrong rewrites user input. + */ +export function coerceArgsToSchema( + args: Record, + schema: { properties?: Record } | undefined +): { + args: Record + resetKeys: string[] + clearedKeys: string[] + undeclaredKeys: string[] +} { + const properties = schema?.properties ?? {} + const clearedKeys: string[] = [] + const { args: declared, undeclaredKeys } = dropUndeclaredArgs(args, schema) + const kept: Record = Object.create(null) + for (const [key, value] of Object.entries(declared)) { + // Declared, but a declaration can still be nothing, and reading `.type` off it throws. + const prop = properties[key] + if ( + prop === undefined || + value == null || + (isReference(value) && !declaresDynMultiselect(prop)) + ) { + kept[key] = value + continue + } + if (declaresDynMultiselect(prop)) { + if (Array.isArray(value)) kept[key] = value + else clearedKeys.push(key) + continue + } + if (!SCALAR_TYPES.has(prop.type) || fitsScalarType(value, prop.type)) { + kept[key] = value + continue + } + const coerced = coerceScalar(value, prop.type) + if (coerced === UNCOERCIBLE) clearedKeys.push(key) + else kept[key] = coerced + } + const { args: result, resetKeys } = enforceDisabledDefaults({ ...kept }, schema) + return { args: result, resetKeys, clearedKeys, undeclaredKeys } +} + +/** + * Every bag of `properties` a declaration can show a value's keys through, including every + * `oneOf` branch rather than the selected one: a secret under a variant nobody opened + * leaves the form just the same. + */ +function declarationBags(prop: any): Record[] { + const bags: Record[] = [] + if (prop?.properties) bags.push(prop.properties) + if (Array.isArray(prop?.oneOf)) + for (const branch of prop.oneOf) if (branch?.properties) bags.push(branch.properties) + return bags +} + +/** + * Apply `visit` to every value whose declaration matches `isLeaf`, at any depth; returning + * `undefined` removes it. Recursive because the form is, so a level left unvisited is one a + * secret can sit at. Descends on the value's shape, never on the declaration's keys: one + * carrying both `items` and `properties` must not route a shape down the other's branch. + */ +function mapLeaves( + value: any, + prop: any, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown, + path: (string | number)[] +): any { + if (value == null || typeof value !== 'object') return value + // A container shaped unlike its declaration is kept rather than dropped, since the widget + // is the one that reports it — so the walk has to reach in through whichever half the + // declaration does carry, or a secret under one leaves the form verbatim. + if (Array.isArray(value)) + return value.map((item, i) => mapLeaves(item, prop?.items ?? prop, isLeaf, visit, [...path, i])) + const bags = declarationBags(prop) + if (bags.length === 0) + return prop?.items ? mapLeaves(value, prop.items, isLeaf, visit, path) : value + // Null prototype, and keyed off the value rather than the declaration: a key is only + // ever rewritten where it already exists, so no branch of a `oneOf` can add one. + const result: Record = Object.assign(Object.create(null), value) + for (const key of Object.keys(result)) { + const declared = bags.filter((bag) => Object.hasOwn(bag, key)).map((bag) => bag[key]) + // Segments, never a joined name: a key can itself hold a dot, and two leaves reported + // under one name let a caller correlating by it take the one for the other. + const keyPath = [...path, key] + // A matching object is a leaf, not a level: a password object is stored whole as a + // single $jsonvar: reference, and a file is one opaque base64 string. + const leaf = declared.find(isLeaf) + if (leaf) { + const mapped = visit(result[key], leaf, keyPath) + if (mapped === undefined) delete result[key] + else result[key] = mapped + continue + } + for (const declaration of declared) + result[key] = mapLeaves(result[key], declaration, isLeaf, visit, keyPath) + } + return { ...result } +} + +export const isSecretProp = (prop: any) => !!prop?.password + +const isFileProp = (prop: any) => + prop?.contentEncoding === 'base64' || prop?.items?.contentEncoding === 'base64' + +function fileMarker(base64: string): string { + const bytes = Math.floor((base64.length * 3) / 4) + return bytes < 1024 * 1024 + ? `` + : `` +} + +/** {@link mapLeaves} over a whole argument object, against a schema that may declare + * nothing to match. Copies either way, for the reason {@link enforceDisabledDefaults} + * copies. */ +export function mapArgLeaves( + args: Record, + schema: { properties?: Record } | undefined, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown +): Record { + return mapLeaves(args ?? {}, { properties: schema?.properties ?? {} }, isLeaf, visit, []) +} + +/** A leaf's path as the lines naming it to a reader read: `creds[0].secret`. */ +const formatArgPath = (path: (string | number)[]): string => + path.reduce( + (acc, segment) => + typeof segment === 'number' + ? `${acc}[${segment}]` + : acc + ? `${acc}.${segment}` + : String(segment), + '' + ) + +/** + * Drop every file argument, so a caller cannot propose file bytes on the user's behalf: + * the field opens empty and the user attaches the file. Bytes a form is prefilled with + * are bytes the stored transcript carries, unbounded, for a value no caller can produce. + * Reports the path of each one removed, or the caller reads the absence as the user having + * deleted the value. + */ +export function stripFileArgs( + args: Record, + schema: { properties?: Record } | undefined, + strippedKeys?: string[] +): Record { + return mapArgLeaves(args, schema, isFileProp, (value, _prop, path) => { + if (value !== undefined) strippedKeys?.push(formatArgPath(path)) + return undefined + }) +} + +/** + * Replace a sensitive value with a fixed marker, for text that leaves the form. A reference is + * kept: it names a variable rather than holding one, and the run page shows the same job's + * arguments that way. An empty field is kept for the reason `processSecretArgs` mints nothing + * for one — marking it would describe a secret the run never carried. + */ +export function redactSecretArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + return mapArgLeaves(args, schema, isSecretProp, (value) => + value == null ? undefined : value === '' || isReference(value) ? value : '' + ) +} + +/** + * Replace every file argument with a marker naming its size. The base64 belongs in the + * job request and nowhere else: rendered it is unreadable, persisted it is unbounded, and + * a file small enough to survive truncation reaches the model whole. + */ +export function redactFileArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + const mark = (value: unknown) => (typeof value === 'string' ? fileMarker(value) : value) + return mapArgLeaves(args, schema, isFileProp, (value) => + Array.isArray(value) ? value.map(mark) : mark(value) + ) +} + export function isWindmillTooBigObject(obj: any): boolean { return ( typeof obj === 'object' && diff --git a/frontend/src/lib/components/markdownProse.ts b/frontend/src/lib/components/markdownProse.ts index bc943aa42b..3f79ab79a5 100644 --- a/frontend/src/lib/components/markdownProse.ts +++ b/frontend/src/lib/components/markdownProse.ts @@ -4,10 +4,14 @@ * call site with layout-only classes (padding, width, bg); anything typographic * belongs here. * - * - 'xs': micro scale for dense secondary panes (chat reasoning blocks) + * - 'xs': micro scale for dense secondary panes (chat reasoning blocks, group notes) * - 'sm': compact chat-bubble scale (assistant messages, flow/app chat, settings) * - 'doc': same rhythm and body size as 'sm', with a taller heading ramp * (lg/base/sm) and semibold headings for document-like surfaces (artifacts) + * + * h1 and h2 step above the body size in every preset, so `#` and `##` read as + * headings rather than bold body text. Below that the tight 'xs' and 'sm' scales + * run out of room, and h3 down differentiates by weight and colour alone. */ // Kept as literal template parts: Tailwind's scanner reads class names verbatim @@ -22,14 +26,14 @@ const base = // One vertical rhythm for sm/doc; heading margins stay per-preset (fixed, not // the plugin's em-based ones) so 'doc' can breathe more between sections. -const rhythm = 'prose-sm leading-snug prose-ul:!pl-6' +const rhythm = 'prose-sm leading-snug' const bodyXs = 'text-primary prose-p:text-primary prose-li:text-primary prose-p:text-xs prose-li:text-xs prose-code:text-xs prose-pre:text-xs prose-table:text-xs' export const markdownProse = { - xs: `${base} prose-sm leading-snug prose-ul:!pl-5 prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-2xs prose-h2:text-2xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, - sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-sm prose-h2:text-xs prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, + xs: `${base} prose-sm leading-snug prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-sm prose-h2:text-xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, + sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-base prose-h2:text-sm prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, doc: `${base} ${rhythm} ${bodyXs} prose-headings:mt-8 prose-headings:mb-2 prose-headings:font-semibold prose-headings:text-emphasis prose-h1:text-lg prose-h2:text-base prose-h3:text-sm prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs prose-pre:bg-transparent prose-pre:p-0` } as const diff --git a/frontend/src/lib/components/mcp/McpConnect.svelte b/frontend/src/lib/components/mcp/McpConnect.svelte index 19b4d32209..1fe1a2fed9 100644 --- a/frontend/src/lib/components/mcp/McpConnect.svelte +++ b/frontend/src/lib/components/mcp/McpConnect.svelte @@ -27,9 +27,12 @@ /** Required: a caller that forgot it would create the connection in whichever * workspace the ui happens to be showing, not the one it operates on. */ workspace: string + /** Off where the surface around it already draws a card — a popover panel — + * so the two do not stack a border and a background on each other. */ + bordered?: boolean } - let { onConnected, onCancel, workspace }: Props = $props() + let { onConnected, onCancel, workspace, bordered = true }: Props = $props() let ws = $derived(workspace) // Any URL is connectable; a suggestion is a shortcut that also pins how the @@ -130,13 +133,6 @@ let canSignIn = $derived( oauthAppReady || (canDiscover && !!$enterpriseLicense && discoveryFoundOAuth !== false) ) - // The action button names the credential, not the outcome, so the path field - // says what clicking it will leave behind. - let pathNote = $derived( - canSignIn && !showToken - ? 'Signing in saves the connection at this path, as an' - : 'The connection is saved at this path, as an' - ) // Why the token field is the only way in, said where the token is asked for. let tokenNote = $derived( needsOauthApp && entry @@ -338,7 +334,11 @@ } -
    +
    Connect an MCP server {#if onCancel} @@ -442,8 +442,12 @@ {/if} + import type { Component } from 'svelte' + import McpIcon from '$lib/components/icons/McpIcon.svelte' + import { twMerge } from 'tailwind-merge' + + /** + * A connected server's mark: the icon Windmill ships for that integration, or the + * MCP logo for a server it has none for — which still says what kind of thing the + * row reaches, where a generic plug did not. + */ + let { + icon, + size = 16, + class: className = '' + }: { icon?: Component; size?: number; class?: string } = $props() + + const px = $derived(`${size}px`) + + + + diff --git a/frontend/src/lib/components/mcp/enabledServers.ts b/frontend/src/lib/components/mcp/enabledServers.ts index b49b788c67..6dbeb997c1 100644 --- a/frontend/src/lib/components/mcp/enabledServers.ts +++ b/frontend/src/lib/components/mcp/enabledServers.ts @@ -1,11 +1,12 @@ -import { createEnabledPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' +import { createPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' /** Which MCP servers the chat may act through, per workspace and per account. A * server's tools both reach an external system and put their descriptions in the * model's context, so one is off until it is turned on; connecting one through the * chat turns it on for the person who connected it. */ -const preference = createEnabledPathsPreference('wm_mcp_enabled') +const preference = createPathsPreference('wm_mcp_enabled', false) -export const enabledMcpPaths = preference.enabledPaths +/** Servers are off by default, so what is stored as on is the whole enabled set. */ +export const enabledMcpPaths = preference.explicitlyEnabledPaths export const isMcpEnabled = preference.isEnabled export const setMcpEnabled = preference.setEnabled diff --git a/frontend/src/lib/components/mcp/iconCache.ts b/frontend/src/lib/components/mcp/iconCache.ts index e3c0f592e2..59ba63adfb 100644 --- a/frontend/src/lib/components/mcp/iconCache.ts +++ b/frontend/src/lib/components/mcp/iconCache.ts @@ -33,6 +33,18 @@ export function cachedProviderKey( return entry.editedAt === editedAt ? entry.key : undefined } +/** + * The stored mark for a path, ignoring `editedAt`: a transcript row has none to + * match against, and nothing acts on a mark. + */ +export function cachedProviderMark( + workspace: string, + path: string +): { key: string | null } | undefined { + const entry = read()[workspace]?.[path] + return entry ? { key: entry.key } : undefined +} + export function rememberProviderKey( workspace: string, path: string, diff --git a/frontend/src/lib/components/mcp/mcpMenu.svelte.ts b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts new file mode 100644 index 0000000000..aea4a7b114 --- /dev/null +++ b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts @@ -0,0 +1,192 @@ +import { List, Plus } from 'lucide-svelte' +import type { Component } from 'svelte' +import { get } from 'svelte/store' +import { ResourceService } from '$lib/gen' +import { workspaceStore } from '$lib/stores' +import { sendUserToast } from '$lib/toast' +import type { Item } from '$lib/utils' +import type { AIChatManager } from '../copilot/chat/AIChatManager.svelte' +import { isMcpEnabled, setMcpEnabled } from './enabledServers' +import { cachedProviderKey, rememberProviderKey } from './iconCache' +import { loadProviderIcon } from './providerIcon' +import McpServerIcon from './McpServerIcon.svelte' + +type Row = { + path: string + editedAt?: string + enabled: boolean + icon?: Component +} + +// A menu is a shortcut, not a directory: past this many the list stops being +// scannable, so the rest are reached through the settings modal rather than dropped. +const MAX_MENU_SERVERS = 8 +// A row whose provider is already cached paints from the cache; the rest cost one +// read each, and a long list stops asking rather than firing a request storm at a +// menu nobody is reading that far down. +const MAX_ICON_LOOKUPS = 20 + +/** + * The chat "+" menu's MCP submenu: one row per connected server, checked when it + * is on, then the way to manage them. Connecting and deleting live in the + * assistant settings modal, which `onManage` opens. + */ +export class McpMenu { + #manager: AIChatManager + #onManage: () => void + #seq = 0 + /** Rows for the workspace named by `#rowsWorkspace`, and meaningless for any other. */ + #rows = $state([]) + #rowsWorkspace: string | undefined = undefined + + constructor(manager: AIChatManager, onManage: () => void) { + this.#manager = manager + this.#onManage = onManage + } + + // A session chat operates on its own (possibly forked) workspace without + // switching `workspaceStore`, and that is the workspace the chat reads the + // enabled set under. Key everything here the same way or a toggle lands under + // a key nothing reads. Read per call rather than derived: the menu is built + // on open, so there is no stale snapshot to keep current between opens. + get #ws(): string | undefined { + return this.#manager.operatingWorkspace ?? get(workspaceStore) ?? undefined + } + + async #load(ws: string) { + const seq = ++this.#seq + try { + const resources = await ResourceService.listResource({ + workspace: ws, + resourceType: 'mcp', + perPage: 100 + }) + if (seq !== this.#seq) return + this.#rows = resources.map((r) => ({ + path: r.path, + editedAt: r.edited_at, + enabled: isMcpEnabled(ws, r.path) + })) + this.#rowsWorkspace = ws + void this.#loadIcons(ws, seq) + } catch { + // The menu's other entries still work; an MCP submenu that failed to load + // is better empty than blocking the whole "+" menu behind an error. + if (seq !== this.#seq) return + this.#rows = [] + this.#rowsWorkspace = ws + } + } + + async #loadIcons(ws: string, seq: number) { + let lookups = 0 + await Promise.all( + this.#rows.map(async (server) => { + let key = cachedProviderKey(ws, server.path, server.editedAt) + if (key === undefined) { + if (lookups >= MAX_ICON_LOOKUPS) return + lookups++ + try { + const resource = await ResourceService.getResource({ + workspace: ws, + path: server.path + }) + key = rememberProviderKey( + ws, + server.path, + (resource.value as { url?: unknown } | undefined)?.url, + server.editedAt + ) + } catch { + return + } + } + const icon = await loadProviderIcon(key) + if (seq !== this.#seq) return + server.icon = icon + }) + ) + } + + #row(path: string) { + return this.#rows.find((s) => s.path === path) + } + + async #toggle(ws: string, path: string, enabled: boolean) { + // A session whose fork is still staged has no workspace of its own yet, so `ws` + // is the PARENT: the selection would be stored under it and quietly stop + // applying the moment the first send commits the fork. + const pendingForkOf = this.#manager.sessionContextResolver?.()?.pendingForkOf + if (pendingForkOf !== undefined) { + sendUserToast( + `This session has not created its workspace yet, so the selection would be stored under "${pendingForkOf}". Send a message first.`, + true + ) + return + } + // Local preference only: nothing to re-read from the API, and the cached + // tool lists stay valid because the servers are unchanged. Checked, like the + // skills submenu: a refused write leaves the chat carrying a different set than + // the check mark shows. + if (!setMcpEnabled(ws, path, enabled)) { + sendUserToast('Could not save the selection for this account.', true) + return + } + const row = this.#row(path) + if (row) row.enabled = enabled + await this.#manager.refreshMcpServers(ws) + } + + /** Loaded on open so the checks are current. */ + async items(closeMenu?: () => void): Promise { + const ws = this.#ws + if (!ws) return [] + // The menu opens on what is already known and refreshes behind it: awaited + // inline it would stall the whole "+" menu, attachments included. Rows for + // another workspace are not "already known" — same path, different server. + if (this.#rowsWorkspace !== ws) { + this.#rows = [] + await this.#load(ws) + } else { + void this.#load(ws) + } + // Enabled first: those are the ones a quick visit is most likely about. + const ordered = [...this.#rows].sort( + (a, b) => Number(b.enabled) - Number(a.enabled) || a.path.localeCompare(b.path) + ) + const shown = ordered.slice(0, MAX_MENU_SERVERS) + const manage = () => { + closeMenu?.() + this.#onManage() + } + // Bound out here because the getters below sit on plain object literals, + // where `this` is the item rather than this menu. + const row = (path: string) => this.#row(path) + return [ + ...shown.map(({ path }) => ({ + displayName: path, + icon: McpServerIcon, + // Getters, not snapshots: the menu stays open across a click, and it has + // to read through the live list rather than the row captured here, since + // a reload replaces every row object and a getter bound to the old one + // would go on reporting the state it was built with. + get iconProps() { + return { icon: row(path)?.icon, size: 14 } + }, + get toggle() { + return row(path)?.enabled ?? false + }, + action: () => this.#toggle(ws, path, !row(path)?.enabled) + })), + ...(ordered.length > shown.length + ? [{ displayName: `Show all ${ordered.length}`, icon: List, action: manage }] + : []), + { + displayName: 'Connect a server', + icon: Plus, + separatorTop: this.#rows.length > 0, + action: manage + } + ] + } +} diff --git a/frontend/src/lib/components/mcp/secretVariable.ts b/frontend/src/lib/components/mcp/secretVariable.ts index aadd9ac0a8..e84a5ac641 100644 --- a/frontend/src/lib/components/mcp/secretVariable.ts +++ b/frontend/src/lib/components/mcp/secretVariable.ts @@ -12,7 +12,7 @@ function mcpTokenDescription(resourcePath: string): string { /** * Store a connection's token at `path`. * - * Disconnecting an MCP server deletes the resource but deliberately keeps its + * Deleting an MCP connection deletes the resource but deliberately keeps its * token variable, because `delete_resource` cascade-deletes every variable the * value references and that credential may still belong to another resource. So * reconnecting the same server lands on an existing path, which is the only case diff --git a/frontend/src/lib/components/mcp/serverMark.test.ts b/frontend/src/lib/components/mcp/serverMark.test.ts new file mode 100644 index 0000000000..7e34d27284 --- /dev/null +++ b/frontend/src/lib/components/mcp/serverMark.test.ts @@ -0,0 +1,45 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { getResourceMock } = vi.hoisted(() => ({ getResourceMock: vi.fn() })) + +vi.mock('$lib/gen', () => ({ ResourceService: { getResource: getResourceMock } })) +vi.mock('./iconCache', () => ({ cachedProviderMark: () => undefined })) +vi.mock('./providerIcon', () => ({ + loadProviderIcon: async (key: string | null) => (key ? `icon:${key}` : undefined), + providerKey: (url: unknown) => + typeof url === 'string' && url.includes('linear') ? 'linear' : null +})) + +import { forgetMcpServerMarks, resolveMcpServerMark } from './serverMark' + +describe('resolveMcpServerMark', () => { + beforeEach(() => { + forgetMcpServerMarks() + getResourceMock.mockReset() + }) + + it('reads a server once and shares the answer', async () => { + getResourceMock.mockResolvedValue({ value: { url: 'https://mcp.linear.app/mcp' } }) + + const marks = await Promise.all([ + resolveMcpServerMark('ws', 'u/admin/linear_mcp'), + resolveMcpServerMark('ws', 'u/admin/linear_mcp') + ]) + + expect(marks.map((m) => m.icon)).toEqual(['icon:linear', 'icon:linear']) + expect(getResourceMock).toHaveBeenCalledTimes(1) + }) + + // A failure must not be memoized: one offline blip would otherwise leave the + // server unmarked in every later row until the page reloads. + it('retries after a failed read', async () => { + getResourceMock.mockRejectedValueOnce(new Error('offline')) + expect(await resolveMcpServerMark('ws', 'u/admin/linear_mcp')).toEqual({}) + + getResourceMock.mockResolvedValue({ value: { url: 'https://mcp.linear.app/mcp' } }) + const mark = await resolveMcpServerMark('ws', 'u/admin/linear_mcp') + + expect(mark.icon).toBe('icon:linear') + expect(getResourceMock).toHaveBeenCalledTimes(2) + }) +}) diff --git a/frontend/src/lib/components/mcp/serverMark.ts b/frontend/src/lib/components/mcp/serverMark.ts new file mode 100644 index 0000000000..c728abdead --- /dev/null +++ b/frontend/src/lib/components/mcp/serverMark.ts @@ -0,0 +1,47 @@ +import type { Component } from 'svelte' +import { ResourceService } from '$lib/gen' +import { cachedProviderMark } from './iconCache' +import { loadProviderIcon, providerKey } from './providerIcon' + +/** Windmill's own icon for a connected server's integration, when it ships one. */ +export type McpServerMark = { icon?: Component } + +// One resolution per server per session, shared by every transcript row naming it — +// a chat can hold dozens of calls against the same server. +const marks = new Map>() + +export function resolveMcpServerMark(workspace: string, path: string): Promise { + const key = `${workspace}:${path}` + let pending = marks.get(key) + if (!pending) { + // A failed read is dropped rather than memoized: one offline blip or a 401 during + // a token refresh would otherwise leave the server unmarked in every later row + // until the page reloads. + pending = load(workspace, path).catch(() => { + marks.delete(key) + return {} + }) + marks.set(key, pending) + } + return pending +} + +/** + * Forget what was resolved, for the settings section to call when it reloads the + * connections: a path can be reconnected to a different provider, and a mark held for + * the life of the page would go on marking new call rows with the old provider's icon. + */ +export function forgetMcpServerMarks() { + marks.clear() +} + +async function load(workspace: string, path: string): Promise { + const cached = cachedProviderMark(workspace, path) + if (cached) return { icon: await loadProviderIcon(cached.key) } + // Deliberately not written back to the shared cache: that entry is keyed by + // `editedAt` for the server list's sake, and storing one from here — where the + // row is a past call and `editedAt` is unknown — would make every list re-read. + const resource = await ResourceService.getResource({ workspace, path }) + const url = (resource.value as { url?: unknown } | undefined)?.url + return { icon: await loadProviderIcon(providerKey(url)) } +} diff --git a/frontend/src/lib/components/meltComponents/Popover.svelte b/frontend/src/lib/components/meltComponents/Popover.svelte index f850e53f6c..6a6b7d681b 100644 --- a/frontend/src/lib/components/meltComponents/Popover.svelte +++ b/frontend/src/lib/components/meltComponents/Popover.svelte @@ -60,6 +60,8 @@ documentationLink?: string | undefined disableFocusTrap?: boolean openFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined + /** Element to focus when the popover closes; defaults to the trigger, `null` leaves focus alone. */ + closeFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined escapeBehavior?: EscapeBehaviorType enableFlyTransition?: boolean onKeyDown?: (e: KeyboardEvent) => void @@ -99,6 +101,7 @@ documentationLink = undefined, disableFocusTrap = false, openFocus = undefined, + closeFocus = undefined, escapeBehavior = 'close', enableFlyTransition = false, onKeyDown = () => {}, @@ -133,6 +136,7 @@ disableFocusTrap: untrack(() => disableFocusTrap), escapeBehavior: untrack(() => escapeBehavior), openFocus: untrack(() => openFocus), + closeFocus: untrack(() => closeFocus), onOpenChange: ({ curr, next }) => { if (curr != next) { dispatch('openChange', next) diff --git a/frontend/src/lib/components/pickerPopularity.test.ts b/frontend/src/lib/components/pickerPopularity.test.ts new file mode 100644 index 0000000000..f9440981ad --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from 'vitest' +import { alphabetical, byPopularity, totalLocalCountsByApp } from './pickerPopularity' + +const order = (names: string[], hub: Record, local: Record = {}) => + [...names].sort(byPopularity(hub, local)) + +describe('byPopularity', () => { + // The tier that stops a filling-up hub from squeezing the workspace's own stack out of + // the ordering: a global pick count grows without bound, a local one does not. + it('leads with what the workspace uses, whatever the hub says', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { stripe: 1 })).toEqual(['stripe', 'slack']) + }) + + it('ranks the used types among themselves by hub picks', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { slack: 1, stripe: 1 })).toEqual([ + 'slack', + 'stripe' + ]) + }) + + it('breaks a hub tie on how much the workspace uses it', () => { + expect(order(['slack', 'stripe'], { slack: 5, stripe: 5 }, { slack: 1, stripe: 2 })).toEqual([ + 'stripe', + 'slack' + ]) + }) + + it('ranks the unused types by hub picks, below every used one', () => { + expect(order(['ably', 'github', 'stripe'], { ably: 900, github: 5 }, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + it('falls back to alphabetical for everything neither signal ranks', () => { + expect(order(['stripe', 'ably', 'github'], { github: 3 })).toEqual(['github', 'ably', 'stripe']) + }) + + it('orders on local usage alone when the hub ranks nothing', () => { + expect(order(['stripe', 'ably', 'github'], {}, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + // The lists render before either signal lands, and one of them arrives in a server-side + // HashMap's iteration order, so the resting comparator has to sort rather than no-op. + it('leaves an alphabetical order with no signal at all', () => { + expect(['stripe', 'ably', 'github'].sort(alphabetical)).toEqual(['ably', 'github', 'stripe']) + }) +}) + +describe('totalLocalCountsByApp', () => { + const HUB = [ + { name: 'discord_webhook', app: 'discord', picks: 0 }, + { name: 'discord_bot_configuration', app: 'discord', picks: 0 }, + { name: 'ms_teams_webhook', app: 'msteams', picks: 0 }, + { name: 'slack', app: 'slack', picks: 0 } + ] + + // The integration pickers list app names, the counts arrive keyed by resource type, and + // the two only usually agree. Without the mapping a workspace whose Discord credential is + // a `discord_webhook` never reaches the used-here tier at all. + it('totals a resource type under the integration it belongs to', () => { + expect(totalLocalCountsByApp({ discord_webhook: 2 }, HUB)).toEqual({ discord: 2 }) + }) + + it('sums the several types one integration can have', () => { + expect( + totalLocalCountsByApp({ discord_webhook: 2, discord_bot_configuration: 1 }, HUB) + ).toEqual({ discord: 3 }) + }) + + // What a workspace-made type, and an unreachable hub, both leave every entry with. + it('keeps a type the hub has no mapping for under its own name', () => { + expect(totalLocalCountsByApp({ c_acme: 1, slack: 2 }, HUB)).toEqual({ c_acme: 1, slack: 2 }) + }) +}) diff --git a/frontend/src/lib/components/pickerPopularity.ts b/frontend/src/lib/components/pickerPopularity.ts new file mode 100644 index 0000000000..2d585ff858 --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.ts @@ -0,0 +1,149 @@ +import { get } from 'svelte/store' +import { ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { isCustomResourceTypeName } from './resourceTypeDisplay' + +/** + * How often something has been picked or used, keyed by integration or resource type name. + * A name the caller lists but this map does not mention counts as zero, which is what an + * unpicked entry and an absent signal both mean. + */ +export type PopularityCounts = Record + +/** + * The signals are read on every picker open, so they are cached briefly; both resolve to an + * empty map rather than rejecting, since an ordering hint is never worth a broken list. + */ +const CACHE_MS = 60_000 + +type HubResourceTypeInfo = { name: string; app: string; picks: number } + +const hubInfoCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + return await ResourceService.listHubResourceTypeInfo({ workspace }) + } catch { + return [] + } + }, + { invalidateMs: CACHE_MS } +) + +const localCountsCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + const counts = await ResourceService.listResourceCountsByType({ workspace }) + return Object.fromEntries(counts.map((c) => [c.resource_type, c.count])) + } catch { + return {} + } + }, + { invalidateMs: CACHE_MS } +) + +/** + * What the hub sees people pick, per resource type. Empty on a hub that counts nothing, + * and on an instance that has switched the hub off — a closed environment must not spend a + * request on hub.windmill.dev just to order a list. + */ +export async function hubResourceTypePicks(workspace: string): Promise { + if (get(disableHubStore)) return {} + const info = await hubInfoCached({ workspace }) + return Object.fromEntries(info.map((rt) => [rt.name, rt.picks])) +} + +/** + * How many resources of each type this workspace holds — the only evidence about this + * particular team. Keyed by resource type, which is what the add-resource drawer lists. + */ +export function localResourceTypeCounts(workspace: string): Promise { + return localCountsCached({ workspace }) +} + +/** + * The same counts totalled per integration, which is what the flow step picker lists. + * + * A type usually shares its integration's name, but often enough it does not: + * `discord_webhook` and `discord_bot_configuration` are both Discord, `ms_teams_webhook` and + * `azure_bot` are both MS Teams. Only the hub knows that, so a workspace whose Discord + * credential is a `discord_webhook` would otherwise read as one that has never touched + * Discord — and since local usage is the leading tier, that decides which half of the list + * the integration lands in, not merely its position within one. + * + * A type the hub has no mapping for counts under its own name, which is the right guess and + * also what an unreachable hub leaves every type with. + */ +export async function localCountsByIntegration(workspace: string): Promise { + const [counts, info] = await Promise.all([ + localCountsCached({ workspace }), + get(disableHubStore) ? Promise.resolve([]) : hubInfoCached({ workspace }) + ]) + return totalLocalCountsByApp(counts, info) +} + +/** The mapping half of {@link localCountsByIntegration}, separated so it can be tested alone. */ +export function totalLocalCountsByApp( + counts: PopularityCounts, + hub: { name: string; app: string }[] +): PopularityCounts { + const appOf = new Map(hub.map((rt) => [rt.name, rt.app])) + const byApp: PopularityCounts = {} + for (const [name, count] of Object.entries(counts)) { + const app = appOf.get(name) ?? name + byApp[app] = (byApp[app] ?? 0) + count + } + return byApp +} + +/** + * Tell the hub a resource type was taken into a workspace, which is what its ranking counts. + * Fire-and-forget: a hub that does not count picks must not be felt by the user who just + * saved a resource. Workspace-made types exist on no hub, so they are not reported. + */ +export function recordHubResourceTypePick(workspace: string, resourceType: string): void { + if (get(disableHubStore)) return + if (!resourceType || isCustomResourceTypeName(resourceType)) return + ResourceService.pickHubResourceType({ workspace, name: resourceType }).catch(() => {}) +} + +/** + * Orders the lists that offer hub content: integrations in the flow step picker, resource + * types in the add-resource drawer. + * + * Four tiers. **Whether this workspace already holds a resource of the type leads**, then the + * hub's pick count, then how many local resources there are, then the name. + * + * Used-here leads rather than merely breaking hub ties because the two counts are on + * incomparable scales: a hub pick count is global and grows without bound, a local count is + * usually single digits. Ranked the other way round, local usage only ever sorts the slice + * where hub counts are equal — which, since they are distinct integers, is just the tail + * that nobody has picked. That reads fine on a hub with few picks and silently stops + * mattering as one fills up, so the ordering would drift away from the workspace's own + * stack with no change to this code. + * + * Within each half the hub decides, so "yours" and "everyone's" are both honoured rather + * than blended with a weighting constant that would need tuning. Alphabetical is the floor, + * and it is where an entry neither signal knows about lands. + */ +export function byPopularity( + hub: PopularityCounts, + local: PopularityCounts +): (a: string, b: string) => number { + const usedHere = (name: string) => ((local[name] ?? 0) > 0 ? 1 : 0) + return (a, b) => + usedHere(b) - usedHere(a) || + (hub[b] ?? 0) - (hub[a] ?? 0) || + (local[b] ?? 0) - (local[a] ?? 0) || + a.localeCompare(b) +} + +/** + * The ordering to hold before either signal has landed: the alphabetical floor, which is + * what `byPopularity` degrades to anyway. + * + * A list has to be sorted by *something* from its first paint — one source of these names + * is a `HashMap` on the server, so leaving them unsorted means hash order, which differs + * between processes. + */ +export const alphabetical = byPopularity({}, {}) diff --git a/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts b/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts index 444e03a0a3..6a18a28e46 100644 --- a/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts +++ b/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts @@ -1,5 +1,6 @@ import { extToLang } from '$lib/editorLangUtils' -import { cleanValueProperties, orderedYamlStringify, replaceFalseWithUndefined } from '$lib/utils' +import { cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' +import { orderedYamlStringify } from '$lib/utils/orderedYaml' // A raw app rendered as a *folder of files* for diffing. Each entry is one // virtual file: real `files` keep their natural path, runnables become diff --git a/frontend/src/lib/components/runs/RunRow.svelte b/frontend/src/lib/components/runs/RunRow.svelte index 27341d7743..959ee2fd06 100644 --- a/frontend/src/lib/components/runs/RunRow.svelte +++ b/frontend/src/lib/components/runs/RunRow.svelte @@ -127,6 +127,8 @@ {/if} {:else if `scheduled_for` in job && job.scheduled_for && forLater(job.scheduled_for)} Waiting executor () + {:else if 'running' in job && job.running && job.suspend} + Suspended (created ) {:else} Waiting executor () {/if} diff --git a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts index 5d2f878bfd..828f060972 100644 --- a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts +++ b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts @@ -18,6 +18,9 @@ import { CancelablePromiseUtils } from '$lib/cancelable-promise-utils' import type { Timeframe } from './timeframes' import { allowWildcards as _allowWildcards, type RunsFilterInstance } from './runsFilter' +// windmill_common::utils::MAX_PER_PAGE: the server silently caps per_page at this value +const MAX_PER_PAGE = 10000 + export function computeJobKinds(jobKindsCat: string | null): string { if (jobKindsCat == 'all') { return '' @@ -75,6 +78,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { let label = $derived(filters?.label ?? null) let worker = $derived(filters?.worker ?? null) let success = $derived(filters?.status ?? null) + let isQueueOnly = $derived(success == 'running' || success == 'suspended' || success == 'waiting') let showSkipped = $derived(filters?.show_skipped ?? false) let resolutionFilter = $derived(filters?.resolved ?? 'all') let showSchedules = $derived(!filters?.job_trigger_kind?.includes('!schedule')) @@ -126,9 +130,14 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { let promise = loadJobsIntern(true) if (perPage > 25) { promise = CancelablePromiseUtils.onTimeout(promise, 4000, () => { - sendUserToast('Loading jobs is taking longer than expected...', 'warning', [ - { label: 'Stream by batches of 25', callback: () => restreamWithSmallBatches() } - ]) + const noStartDate = timeframe?.computeMinMax().minTs == null + sendUserToast( + (success == 'failure' || success == 'canceled') && noStartDate + ? `Loading ${success == 'failure' ? 'failed' : 'canceled'} jobs with no start date scans the full job history. Set a time range to speed it up.` + : 'Loading jobs is taking longer than expected...', + 'warning', + [{ label: 'Stream by batches of 25', callback: () => restreamWithSmallBatches() }] + ) }) } promise = CancelablePromiseUtils.finallyDo(promise, () => { @@ -191,21 +200,40 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loadingExtra = false } + // Mirrors when list_completed_jobs_query sorts by completed_at. A created_at cursor does not + // bound that index scan, so each batch would rescan from the newest job, and skip jobs created + // after the cursor but completed before it. + function sortsByCompletedAt(minTs: string | null, maxTs: string | null): boolean { + return minTs != null || maxTs != null || success == 'failure' || success == 'canceled' + } + function loadExtraJobsBatch(batchSize: number): CancelablePromise { if (!jobs || jobs.length === 0) { lastFetchWentToEnd = true return CancelablePromiseUtils.pure(undefined as void) } - const lastJob = jobs[jobs.length - 1] - const ts = lastJob.created_at - if (!ts) { + const { minTs, maxTs } = timeframe?.computeMinMax() ?? { minTs: null, maxTs: null } + const byCompletedAt = + jobs[jobs.length - 1].type === 'CompletedJob' && sortsByCompletedAt(minTs, maxTs) + const sortKey = (j: Job) => + byCompletedAt ? (j.type === 'CompletedJob' ? j.completed_at : undefined) : j.created_at + const cursorTs = sortKey(jobs[jobs.length - 1]) + if (!cursorTs) { lastFetchWentToEnd = true return CancelablePromiseUtils.pure(undefined as void) } - const cursorTs = new Date(new Date(ts).getTime() - 1).toISOString() - const minTs = timeframe?.computeMinMax().minTs ?? null + // Inclusive cursor at the API's microsecond precision: jobs sharing the boundary timestamp (e.g. + // a bulk cancel) are refetched rather than skipped, and the page grows by those already listed, + // up to the server's MAX_PER_PAGE. Once the listed part of the group fills that cap, the cursor + // steps just below the group, dropping its remainder instead of ending the list early. + const tied = jobs.filter((j) => sortKey(j) === cursorTs).length + const stepOver = tied >= MAX_PER_PAGE + const cursor = stepOver ? new Date(new Date(cursorTs).getTime() - 1).toISOString() : cursorTs + const pageSize = stepOver ? batchSize : Math.min(batchSize + tied, MAX_PER_PAGE) return CancelablePromiseUtils.map( - fetchJobs(null, minTs, undefined, cursorTs, batchSize), + byCompletedAt + ? fetchJobs(cursor, minTs, undefined, undefined, pageSize) + : fetchJobs(null, minTs, undefined, cursor, pageSize), (olderJobs) => { jobs = updateWithNewJobs(olderJobs ?? [], jobs ?? []) if (extendedJobs) { @@ -213,7 +241,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { extendedJobs = extendedJobs } computeCompletedJobs() - lastFetchWentToEnd = (olderJobs?.length ?? 0) < batchSize + lastFetchWentToEnd = (olderJobs?.length ?? 0) < pageSize loading = false } ) @@ -230,7 +258,6 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loadingFetch = true let scriptPathStart = folder == null || folder === '' ? undefined : `f/${folder}/` let scriptPathExact = path == null || path === '' ? undefined : path - let isQueueOnly = success == 'running' || success == 'suspended' || success == 'waiting' let isCompletedOnly = success == 'success' || success == 'failure' || success == 'canceled' let promise = JobService.listJobs({ workspace: currentWorkspace, @@ -289,7 +316,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { }) promise = CancelablePromiseUtils.catchErr(promise, (e) => { if (e instanceof CancelError) return CancelablePromiseUtils.err(e) - sendUserToast('There was an issue loading jobs, see browser console for more details', true) + sendUserToast(`Could not load jobs: ${e.body ?? e.message}`, true) console.error(e) return CancelablePromiseUtils.pure([]) }) @@ -394,6 +421,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { overrideBatchSize?: number ): CancelablePromise { const { minTs, maxTs } = timeframe?.computeMinMax() ?? { minTs: null, maxTs: null } + listLoadedAt = new Date(Date.now() - 5 * 60_000).toISOString() if (shouldGetCount) { getCount() } @@ -529,6 +557,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { } let lastQueueTs: string | undefined = undefined + let listLoadedAt: string | null = null async function syncer() { if (loadingFetch) { @@ -575,7 +604,15 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loading = true let newJobs: Job[] if (concurrencyKey == null || concurrencyKey === '') { - newJobs = await fetchJobs(maxTs, minTs ?? completedTs, queueTs) + // With no completed job to anchor on, each refresh would repeat the initial + // unbounded scan of completed jobs, possibly the one that just timed out. Not for + // queue-only views: fetchJobs turns this into the queue's created_at bound, hiding + // older jobs that suspend or come due. The margin absorbs browser/database skew. + newJobs = await fetchJobs( + maxTs, + minTs ?? completedTs ?? (isQueueOnly ? null : listLoadedAt), + queueTs + ) } else { // Obscured jobs have no ids, so we have to do the full request extendedJobs = await fetchExtendedJobs(concurrencyKey, maxTs, minTs ?? completedTs) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index a117d2769c..084ad9d492 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -427,7 +427,7 @@ path = `/apps/get/${e.path}` break case 'raw_app': - path = `/raw_apps/get/${e.path}` + path = `/apps_raw/get/${e.path}` break default: path = '/' diff --git a/frontend/src/lib/components/secretArgUtils.test.ts b/frontend/src/lib/components/secretArgUtils.test.ts new file mode 100644 index 0000000000..d455b4ad62 --- /dev/null +++ b/frontend/src/lib/components/secretArgUtils.test.ts @@ -0,0 +1,130 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const created: { path: string; value: string; is_secret?: boolean }[] = [] + +vi.mock('$lib/gen', () => ({ + VariableService: { + createVariable: vi.fn(async ({ requestBody }: any) => { + created.push(requestBody) + return requestBody.path + }) + } +})) + +vi.mock('$lib/stores', async () => { + const { writable } = await import('svelte/store') + return { workspaceStore: writable('test-ws'), userStore: writable({ username: 'ada' }) } +}) + +import { processSecretArgs } from './secretArgUtils' + +describe('processSecretArgs', () => { + beforeEach(() => (created.length = 0)) + + const schema = { + properties: { + token: { type: 'string', password: true }, + creds: { type: 'object', password: true, properties: { user: { type: 'string' } } }, + nested: { type: 'object', properties: { inner: { type: 'string', password: true } } }, + plain: { type: 'string' } + } + } as any + + // Nothing else turns a proposed secret into a reference when no form mounts, so a literal + // left alone here is a plaintext credential stored on the job for anyone who can see it. + it('mints a reference for a literal at every level, leaving other arguments alone', async () => { + const out = await processSecretArgs( + { token: 'hunter2', creds: { user: 'ada' }, nested: { inner: 'deep' }, plain: 'kept' }, + schema + ) + expect(out.token).toMatch(/^\$var:u\/ada\/secret_arg\//) + expect(out.creds).toMatch(/^\$jsonvar:u\/ada\/secret_arg\//) + expect(out.nested.inner).toMatch(/^\$var:u\/ada\/secret_arg\//) + expect(out.plain).toBe('kept') + // The object goes into the variable as JSON, which is what `$jsonvar:` parses back. + expect(created.map((c) => c.value).sort()).toEqual(['deep', 'hunter2', '{"user":"ada"}']) + expect(created.every((c) => c.is_secret)).toBe(true) + }) + + it('leaves a reference the caller already named alone', async () => { + const out = await processSecretArgs( + { token: '$var:f/team/api_token', creds: '$jsonvar:u/ada/existing' }, + schema + ) + expect(out).toEqual({ token: '$var:f/team/api_token', creds: '$jsonvar:u/ada/existing' }) + expect(created).toEqual([]) + }) + + // `$var:` hands the job the variable's text; a field declaring an object needs it parsed, + // which is the same variable read the other way rather than a secret the caller cannot see. + it('reads a plain variable as JSON where the field cannot hold a string', async () => { + const out = await processSecretArgs({ creds: '$var:u/ada/stripe' }, schema) + expect(out.creds).toBe('$jsonvar:u/ada/stripe') + expect(created).toEqual([]) + }) + + // Reached by every run form in the product, not only the ones a chat opens. + it('leaves an absent, null or empty secret alone', async () => { + expect(await processSecretArgs({ token: null, plain: 'kept' }, schema)).toEqual({ + token: null, + plain: 'kept' + }) + expect(await processSecretArgs({ token: '', plain: 'kept' }, schema)).toEqual({ + token: '', + plain: 'kept' + }) + expect(created).toEqual([]) + }) + + // A property name can itself contain a dot. Reported under one label these two leaves + // would share a mint, and the flat field would run on the nested field's secret. + it("tells apart a key that spells another key's path", async () => { + const out = await processSecretArgs({ 'db.password': 'FLAT', db: { password: 'NESTED' } }, { + properties: { + 'db.password': { type: 'string', password: true }, + db: { + type: 'object', + properties: { password: { type: 'string', password: true } } + } + } + } as any) + const flat = out['db.password'].slice('$var:'.length) + const nested = out.db.password.slice('$var:'.length) + expect(flat).not.toBe(nested) + expect(created.find((c) => c.path === flat)?.value).toBe('FLAT') + expect(created.find((c) => c.path === nested)?.value).toBe('NESTED') + }) + + // Callers bind a form that stays editable while the mints are in flight, and a leaf is + // addressed by its path: a row moved between the two walks would take the other row's + // reference and the job would run it on the wrong credentials. + it('ignores the caller mutating the arguments while minting', async () => { + const rows = { + creds: [ + { name: 'alpha', secret: 'FIRST' }, + { name: 'beta', secret: 'SECOND' } + ] + } + const arraySchema = { + properties: { + creds: { + type: 'array', + items: { properties: { name: {}, secret: { password: true } } } + } + } + } as any + + const pending = processSecretArgs(rows, arraySchema) + rows.creds.reverse() + const out = await pending + + // Keyed by the row's own name, not its index: a substitution by position lands the + // first-minted reference on index 0 either way. + const secretOf = (name: string) => { + const row = out.creds.find((c: any) => c.name === name) + return created.find((c) => c.path === row.secret.slice('$var:'.length))?.value + } + expect(secretOf('alpha')).toBe('FIRST') + expect(secretOf('beta')).toBe('SECOND') + }) +}) diff --git a/frontend/src/lib/components/secretArgUtils.ts b/frontend/src/lib/components/secretArgUtils.ts index 91101d4816..a330d3bcd9 100644 --- a/frontend/src/lib/components/secretArgUtils.ts +++ b/frontend/src/lib/components/secretArgUtils.ts @@ -3,11 +3,54 @@ import { VariableService } from '$lib/gen' import { get } from 'svelte/store' import { userStore, workspaceStore } from '$lib/stores' import { generateRandomString } from '$lib/utils' +import { stateSnapshot } from '$lib/stateSnapshot.svelte' +import { isSecretProp, mapArgLeaves } from './job_args' + +/** Where a caller's own ephemeral secrets live, so a field can tell one it minted from a + * workspace variable someone linked by hand. */ +export function ephemeralSecretPrefix(username: string): string { + return `u/${username}/secret_arg/` +} /** - * Process args before job submission: for non-string fields marked as password/sensitive, - * create ephemeral secret variables and replace values with $jsonvar:path references. - * String password fields are already handled by PasswordArgInput (uses $var:). + * Mint the ephemeral secret variable a sensitive argument is submitted as, and return its path. + * It expires on its own, so a run that is abandoned leaves no permanent secret behind. + */ +export async function mintEphemeralSecret( + workspace: string, + username: string, + value: string +): Promise { + const path = ephemeralSecretPrefix(username) + generateRandomString(12) + await VariableService.createVariable({ + workspace, + requestBody: { + value, + is_secret: true, + path, + description: 'Ephemeral secret variable', + expires_at: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7).toISOString() + } + }) + return path +} + +/** `$var:` hands the job the variable's text and `$jsonvar:` hands it the parsed value, so a + * field that cannot hold a string needs the second one whichever the caller named. */ +function referencePrefix(prop: any, value: unknown): '$var:' | '$jsonvar:' { + return typeof value === 'string' && prop?.type !== 'object' && prop?.type !== 'array' + ? '$var:' + : '$jsonvar:' +} + +/** + * Turn every sensitive argument into a reference before the job is submitted: a plaintext value + * is minted into an ephemeral secret variable, so what is stored on the job — and readable by + * anyone who can see its run — names a secret instead of holding one. + * + * The single place that decides how a secret reaches a job: {@link PasswordArgInput} mints + * through it while the user types, and a run the autonomy posture starts without a form calls it + * in the widget's stead. */ export async function processSecretArgs( args: Record, @@ -24,29 +67,53 @@ export async function processSecretArgs( const username = (user.username ?? user.email)?.split('@')[0] if (!username) return args - const userPrefix = `u/${username}/secret_arg/` - const result = { ...args } + // Detached from the caller: every one binds a form that stays editable across the awaits + // below, and the two walks address a leaf by its path — an array reordered between them + // would hand a row the reference minted for another row's secret. + args = stateSnapshot(args) - for (const [key, prop] of Object.entries(schema.properties)) { - if (!prop.password) continue - if (prop.type !== 'object') continue // only object types; strings handled by PasswordArgInput - if (result[key] == null || result[key] === undefined) continue - if (typeof result[key] === 'string' && result[key].startsWith('$jsonvar:')) continue // already processed + // A value that already names a variable is one; anything else is the secret itself. An empty + // field holds nothing to mint, and ArgInput synthesises '' for every untouched string. + const holdsSecret = (value: unknown) => + value != null && + value !== '' && + !( + typeof value === 'string' && + (value.startsWith('$var:') || value.startsWith('$jsonvar:') || value.startsWith('$res:')) + ) - const path = userPrefix + generateRandomString(12) - await VariableService.createVariable({ + // Collected first and substituted after, because the walk is synchronous and minting is not. + // Keyed by the whole path the walk reports, which is what tells two same-named leaves apart. + const pending: { key: string; prop: any; value: unknown }[] = [] + mapArgLeaves(args, schema as any, isSecretProp, (value, prop, path) => { + if (holdsSecret(value)) pending.push({ key: JSON.stringify(path), prop, value }) + return value + }) + + const minted = new Map() + for (const { key, prop, value } of pending) { + const reference = referencePrefix(prop, value) + const variable = await mintEphemeralSecret( workspace, - requestBody: { - value: JSON.stringify(result[key]), - is_secret: true, - path, - description: 'Ephemeral secret variable', - expires_at: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7).toISOString() - } - }) - result[key] = '$jsonvar:' + path + username, + reference === '$var:' ? String(value) : JSON.stringify(value) + ) + minted.set(key, reference + variable) } - return result + return mapArgLeaves(args, schema as any, isSecretProp, (value, prop, path) => { + const replacement = minted.get(JSON.stringify(path)) + if (replacement !== undefined) return replacement + // A plain variable named for a field that cannot hold a string: the caller meant that + // variable's contents, which is the same secret read the way the field needs it. + if ( + typeof value === 'string' && + value.startsWith('$var:') && + referencePrefix(prop, value) === '$jsonvar:' + ) { + return '$jsonvar:' + value.slice('$var:'.length) + } + return value + }) } diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index bf26605ac2..3dee7dd1b3 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -19,6 +19,7 @@ } from './previewRouter' import { withMenuHidden } from './sessionMode.svelte' import ArtifactViewer from '../copilot/chat/artifacts/ArtifactViewer.svelte' + import RunFormPreviewSlot from './RunFormPreviewSlot.svelte' import { setOverlayHost } from '../common/overlayHost.svelte' let { @@ -327,6 +328,20 @@
    This artifact is no longer available.
    {/if}
    +{:else if slot.kind === 'runform' && mounted} +
    + + {#if runtime && overlayHostEl} + + {/if} +
    {:else if mounted}
    Triggers to deploy
    +
    +
    + + {#if trigger.isPrimary} + + {/if} +
    + +
    + +
    +
    +
    + {#if permission === 'deploy'} +
    + toggleTrigger(trigger, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
    + {:else if permission === 'admin-only'} + Admin only + {:else if permission === 'invalid-config'} + Invalid config + {/if} +
    Agents to deploy
    +
    + +
    +
    + + + {agent.path} + + {#if agent.noDeployed} + Never deployed + {/if} +
    + {#if agent.noDeployed && !isSelectedAgent} + + + Never deployed, so the flow will not run until this agent is deployed. + + {/if} +
    +
    +
    + {#if permission.state === 'deploy'} +
    + toggleAgent(agent, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
    + {:else if permission.state === 'read-only'} + + Read-only + + {:else} + Invalid config + {/if} +
    Triggers to deploy
    -
    -
    - - {#if trigger.isPrimary} - - {/if} -
    -
    - -
    -
    -
    - {#if permission === 'deploy'} -
    - toggleTrigger(trigger, e.detail)} - > - {#snippet children({ item })} - - - {/snippet} - -
    - {:else if permission === 'admin-only'} - Admin only - {:else if permission === 'invalid-config'} - Invalid config - {/if} -
    - No draft triggers found -