diff --git a/.agents/skills/local-review-codex/SKILL.md b/.agents/skills/local-review-codex/SKILL.md index cc932f7a4b..ec1277fe3a 100644 --- a/.agents/skills/local-review-codex/SKILL.md +++ b/.agents/skills/local-review-codex/SKILL.md @@ -1,6 +1,6 @@ --- name: local-review-codex -description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy, model, and reasoning effort as the codex-pr-review GitHub action. +description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy and reasoning effort as the codex-pr-review GitHub action, on a newer model. --- # Local Codex Review (pre-push) @@ -11,17 +11,18 @@ before the PR exists. Use this before `git push` on a non-trivial change. **Correspondence with CI** — identical: - Policy: `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test coverage). -- Model: `gpt-5.6-sol`, `model_reasoning_effort="xhigh"`. +- Reasoning effort: `model_reasoning_effort="xhigh"`. - Output: markdown starting with `## Codex Review`, findings tagged P0 / P1 / P2 with file:line. **Differences from CI** — local-only: +- Model is `gpt-6-astra`; CI stays on `gpt-5.6-sol`. Not an oversight to reconcile: `gpt-6-astra` is confirmed on the ChatGPT auth `codex login` uses locally, while CI authenticates with `OPENAI_API_KEY` (`codex-pr-review.yml` prefers it over `CODEX_AUTH_JSON`) and that tier is unverified for the model. Move CI once API access is confirmed, or once CI switches to `CODEX_AUTH_JSON`. - Scope is the current branch vs `main` at the merge-base, **including uncommitted changes** (CI reviews a pushed PR diff). - Sandbox is `read-only` (CI uses `danger-full-access` on an ephemeral runner). Codex reads the diff and files but cannot modify your working tree. - Fresh context is inherent: `codex exec` is a separate cold process, so it does not anchor on the current chat session — the same reason `local-review` insists on a subagent. ## Prerequisites -- `codex` CLI **>= 0.144.1** installed and authed (`codex login` or `OPENAI_API_KEY`). Older CLIs reject `gpt-5.6-sol` with "requires a newer version of Codex". Upgrade with `npm install --global @openai/codex@0.144.1` (may need `sudo` for a global install). Keep this in sync with the pin in `.github/workflows/codex-pr-review.yml`. +- `codex` CLI **>= 0.153.4** installed and authed via `codex login` (an `OPENAI_API_KEY` in the environment takes priority and may not reach `gpt-6-astra` — see the model note above). Older CLIs reject the model with "requires a newer version of Codex"; `run.sh` checks the version up front. Upgrade with `npm install --global @openai/codex@0.153.4` (may need `sudo` for a global install). This matches the pin in `.github/workflows/codex-pr-review.yml` — the CLI version is the same on both sides, only the model differs. - `git fetch` the base ref if it's stale, so the merge-base is accurate. ## Run diff --git a/.agents/skills/local-review-codex/run.sh b/.agents/skills/local-review-codex/run.sh index d6491099c2..948d3820eb 100755 --- a/.agents/skills/local-review-codex/run.sh +++ b/.agents/skills/local-review-codex/run.sh @@ -1,21 +1,44 @@ #!/usr/bin/env bash # Local Codex review — mirrors the .github/workflows/codex-pr-review.yml CI job, # but scoped to this branch's unpushed work (committed + uncommitted) so you can -# review before pushing. Same policy (REVIEW.md), same model (gpt-5.6-sol) and -# reasoning effort (xhigh) as CI. Runs read-only: Codex cannot modify your tree. +# review before pushing. Same policy (REVIEW.md) and reasoning effort (xhigh) as CI. +# +# The model deliberately differs from CI: gpt-6-astra is confirmed available on the +# ChatGPT auth `codex login` uses here, but CI authenticates with OPENAI_API_KEY and +# that tier is unverified for it, so codex-pr-review.yml stays on gpt-5.6-sol. # # Usage: run.sh [BASE_REF] (BASE_REF defaults to "main") set -euo pipefail +MODEL="gpt-6-astra" +CODEX_MIN="0.153.4" + BASE_REF="${1:-main}" REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" if ! command -v codex >/dev/null 2>&1; then - echo "codex CLI not found. Install with: npm install --global @openai/codex@0.144.1" >&2 + echo "codex CLI not found. Install with: npm install --global @openai/codex@$CODEX_MIN" >&2 exit 1 fi +# Older CLIs reject the model with an error that never names the CLI version as the +# cause, so check it up front rather than letting the exec fail opaquely. The `|| true` +# keeps an unrecognised --version format from aborting under `set -e`: an unparseable +# version means "cannot tell", which must fall through to the exec, not kill the review. +CODEX_VER="$(codex --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" +if [ -n "$CODEX_VER" ] && [ "$(printf '%s\n%s\n' "$CODEX_MIN" "$CODEX_VER" | sort -V | head -1)" != "$CODEX_MIN" ]; then + echo "codex $CODEX_VER is too old for $MODEL (need >= $CODEX_MIN). Upgrade with: npm install --global @openai/codex@$CODEX_MIN" >&2 + exit 1 +fi + +# codex prefers OPENAI_API_KEY over the ChatGPT credentials `codex login` stores, and +# that tier is not confirmed for $MODEL — the resulting failure names the model, not the +# auth that selected it. +if [ -n "${OPENAI_API_KEY:-}" ]; then + echo "warning: OPENAI_API_KEY is set and takes priority over 'codex login' credentials; $MODEL may be unavailable on that tier." >&2 +fi + # Resolve the base to a concrete commit, preferring a local ref but falling back to # the remote-tracking ref — checkouts (CI, single-branch clones) often have only # origin/main, not a local main. @@ -80,7 +103,7 @@ EOF codex exec \ -C "$REPO_ROOT" \ - -m gpt-5.6-sol \ + -m "$MODEL" \ -c 'model_reasoning_effort="xhigh"' \ -s read-only \ -o "$OUT" \ diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 2f23b21eb8..72bb2b9d8c 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -12,6 +12,7 @@ sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath} sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json +sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml diff --git a/.github/change-versions.sh b/.github/change-versions.sh index ffb73f6180..1772ddbc89 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -13,6 +13,7 @@ sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/o sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/jsr.json +sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/windmill-yaml-validator/package.json sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml @@ -33,3 +34,5 @@ cd ${root_dirpath}/frontend && npm i --package-lock-only --ignore-scripts # The CLI installs this package on every `bun install`, which would otherwise rewrite the # lockfile's version and leave a dirty tree. cd ${root_dirpath}/windmill-yaml-validator && npm i --package-lock-only --ignore-scripts + +cd ${root_dirpath}/chat-sdk && npm i --package-lock-only --ignore-scripts diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 26b2d9aae8..7a3882df7c 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -219,7 +219,7 @@ jobs: - name: Install Codex CLI if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - run: npm install --global @openai/codex@0.144.1 + run: npm install --global @openai/codex@0.153.4 - name: Configure Codex auth if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' diff --git a/.github/workflows/frontend-check.yml b/.github/workflows/frontend-check.yml index a675c17444..178ed53dc5 100644 --- a/.github/workflows/frontend-check.yml +++ b/.github/workflows/frontend-check.yml @@ -9,6 +9,8 @@ on: push: paths: - "frontend/**" + # The flow chat compiles the chat SDK's source in (svelte.config.js alias). + - "chat-sdk/src/**" - ".github/workflows/frontend-check.yml" jobs: diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index a41bd80854..7d30265c74 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -17,6 +17,17 @@ jobs: - run: cd typescript-client && ./publish.sh --access public && cd .. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + publish_chat_sdk: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + - run: cd chat-sdk && npm ci && npm run build && npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_cli: runs-on: ubicloud-standard-8 steps: diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index efcbcd6ec1..6f7d79c6af 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -32,6 +32,25 @@ jobs: working-directory: ./typescript-client run: bun test --timeout 120000 tests/ + chat-sdk: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + + - name: Run tests + working-directory: ./chat-sdk + run: npm ci && npm run check && bun test + python-client: runs-on: ubuntu-latest steps: diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5dc2004c6f..2d6bef08fe 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.804.0" + ".": "1.812.0" } diff --git a/AGENTS.md b/AGENTS.md index 71c59b479d..8a63c9828a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,14 +30,17 @@ Open-source platform for internal tools, workflows, API integrations, background reaches the DB only through the API, so `Connection::Http` paths are never taken by a plain `cargo run`; a normal build cannot start one at all. - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow +- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation + scope, how OAuth login matches `login_type`, and that every superadmin route refuses `$WM_TOKEN`. + Read before designing anything that creates users, tokens or sessions. - **Product telemetry**: `docs/feature-telemetry.md` — when to instrument a new feature with `feature_usage`, and the four-step recipe. An unregistered `(feature, kind)` pair is dropped silently, so frontend-only instrumentation records nothing. - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. - **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead. -- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy, `gpt-5.6-sol`, `xhigh` reasoning; requires the `codex` CLI >= 0.144.1. -- **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc` +- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy and `xhigh` reasoning, on `gpt-6-astra` rather than the action's `gpt-5.6-sol`; requires the `codex` CLI >= 0.153.4. +- **Domain guides**: `.claude/skills/native-trigger/` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfb0edad..49005a22a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,202 @@ # Changelog +## [1.812.0](https://github.com/windmill-labs/windmill/compare/v1.811.1...v1.812.0) (2026-09-15) + + +### Features + +* add per-route CORS origin allowlist for HTTP triggers ([#10833](https://github.com/windmill-labs/windmill/issues/10833)) ([d8d7332](https://github.com/windmill-labs/windmill/commit/d8d7332eb6d92f7a55b82890de5de4196039b40d)) +* **ai-sessions:** share session artifacts with the workspace by link ([#11115](https://github.com/windmill-labs/windmill/issues/11115)) ([57a134e](https://github.com/windmill-labs/windmill/commit/57a134e2de18e27b3c1d3066a60b892af1089b30)) +* **cli:** list, get and restore trashed items with wmill trash ([#11125](https://github.com/windmill-labs/windmill/issues/11125)) ([a95e950](https://github.com/windmill-labs/windmill/commit/a95e950529f6f01a220e09d322d5a4fb507ea694)) +* dynamic AI agent toolsets ([#11050](https://github.com/windmill-labs/windmill/issues/11050)) ([a78beff](https://github.com/windmill-labs/windmill/commit/a78beff743f6bb289805c263a8d32515bea9688f)) +* **git-sync:** gate GitHub PRs on Windmill CI test results (WIN-2051) ([#10096](https://github.com/windmill-labs/windmill/issues/10096)) ([80eba80](https://github.com/windmill-labs/windmill/commit/80eba80d6ed51753cfaa67310f1a0f5dd5ce0484)) +* pre-approved cloud accounts: login links, OAuth adoption, setup, and the trial bridge ([#10875](https://github.com/windmill-labs/windmill/issues/10875)) ([91e6dc3](https://github.com/windmill-labs/windmill/commit/91e6dc39ce795fafc2bed0d799b62c9880fd6430)) +* run a flow step test through the chat's argument form ([#11114](https://github.com/windmill-labs/windmill/issues/11114)) ([5d32b61](https://github.com/windmill-labs/windmill/commit/5d32b6106788b665e4752fcac63ff1ef457d604e)) +* store resource type display names and label hub integrations ([#11113](https://github.com/windmill-labs/windmill/issues/11113)) ([42f4896](https://github.com/windmill-labs/windmill/commit/42f489685bc87a8479818175c87b5a21b0c17998)) +* windmill-chat sdk for chat-mode flows in external frontends and raw apps ([#11117](https://github.com/windmill-labs/windmill/issues/11117)) ([e8c02c0](https://github.com/windmill-labs/windmill/commit/e8c02c04cdb1a3f199f3f0a8b53a839a53d4a1d9)) + + +### Bug Fixes + +* **ai-chat:** hide other users' MCP servers from the chat unless shared ([#11112](https://github.com/windmill-labs/windmill/issues/11112)) ([244ec13](https://github.com/windmill-labs/windmill/commit/244ec132914a6e689d7d79da9cf2cb39f920aaef)) +* **cli:** say where a sync push deleted variable or resource went ([#10851](https://github.com/windmill-labs/windmill/issues/10851)) ([d54a66f](https://github.com/windmill-labs/windmill/commit/d54a66f15c09c34f7a2b45c2e6e9649807a19265)) +* **cli:** stage a rewritten shared lockfile on git-sync deploy push ([#11126](https://github.com/windmill-labs/windmill/issues/11126)) ([75ee497](https://github.com/windmill-labs/windmill/commit/75ee497011dca076de0923ded9da8e23e08bfb84)) +* **flows:** stop re-evaluating skip_if once a loop is in progress ([#11008](https://github.com/windmill-labs/windmill/issues/11008)) ([56e21bc](https://github.com/windmill-labs/windmill/commit/56e21bce832182528562688acd13ec416e01ebfc)) +* **git-sync:** run auto-pull as the admin who enabled it ([#11121](https://github.com/windmill-labs/windmill/issues/11121)) ([69e6efd](https://github.com/windmill-labs/windmill/commit/69e6efd875e020779ea115c096331da8eae2ffe7)) +* keep a script draft's password marking through the chat's run form ([#11110](https://github.com/windmill-labs/windmill/issues/11110)) ([56dd940](https://github.com/windmill-labs/windmill/commit/56dd940e34b146c3ca25de7959b4fb618308eb86)) +* **python:** parse wheel RECORD paths as RFC 4180 csv fields ([#11133](https://github.com/windmill-labs/windmill/issues/11133)) ([9696308](https://github.com/windmill-labs/windmill/commit/96963080f1711192fe1d9bc4142c1710511504d4)) +* re-attach flow chat to the same job on SSE timeout instead of re-running it ([#11122](https://github.com/windmill-labs/windmill/issues/11122)) ([94c548c](https://github.com/windmill-labs/windmill/commit/94c548cd5dc43131e0471b0edabe496dff245862)) +* set the enclosing span's trace context on exported log records ([#11123](https://github.com/windmill-labs/windmill/issues/11123)) ([d0cac08](https://github.com/windmill-labs/windmill/commit/d0cac0807f1b6f4fc76d6e5f7e27e03d30abec8d)) +* skip instance group members that are not email addresses ([#11128](https://github.com/windmill-labs/windmill/issues/11128)) ([e3e638f](https://github.com/windmill-labs/windmill/commit/e3e638f7f587f0ee090d06436575b65a0860a855)) +* wake a WAC parent from every path that completes its child ([#11119](https://github.com/windmill-labs/windmill/issues/11119)) ([0b1e9c0](https://github.com/windmill-labs/windmill/commit/0b1e9c0dda2ae55c56b1e0de5c0419b4511b973f)) + +## [1.811.1](https://github.com/windmill-labs/windmill/compare/v1.811.0...v1.811.1) (2026-09-13) + + +### Bug Fixes + +* check kafka trigger topics against a set, not a one-pass iterator ([#11108](https://github.com/windmill-labs/windmill/issues/11108)) ([bf4fa2b](https://github.com/windmill-labs/windmill/commit/bf4fa2b174b8d4a5897b2aa0d108480442bd0e18)) +* let the hub_sync job read the uid and hub_base_url settings ([#11106](https://github.com/windmill-labs/windmill/issues/11106)) ([45102c8](https://github.com/windmill-labs/windmill/commit/45102c82659d86ca5ec6fd3aee57232f2348736c)) + +## [1.811.0](https://github.com/windmill-labs/windmill/compare/v1.810.0...v1.811.0) (2026-09-12) + + +### Features + +* make snowflake_oauth work as a dbt warehouse on every engine ([#11095](https://github.com/windmill-labs/windmill/issues/11095)) ([9fc50a2](https://github.com/windmill-labs/windmill/commit/9fc50a23fb75cb541c481247b7b25c206fb06d36)) + + +### Bug Fixes + +* accept any hub version of the git sync script in the token check ([#11099](https://github.com/windmill-labs/windmill/issues/11099)) ([670628b](https://github.com/windmill-labs/windmill/commit/670628b300ab119363adb5496ebfe3c6ccd80063)) +* bring back Publish to Hub for scripts ([#11097](https://github.com/windmill-labs/windmill/issues/11097)) ([864e5f0](https://github.com/windmill-labs/windmill/commit/864e5f02ec1c2dd16c74524f551f44485df10a20)) +* bundle deployed bun scripts whose only pin is on a dynamic import ([#11096](https://github.com/windmill-labs/windmill/issues/11096)) ([4afb9aa](https://github.com/windmill-labs/windmill/commit/4afb9aa677ac11d22e22e54bb6fd7881378b7005)) +* clear a stale git auto-pull failure and show the status time ([#11100](https://github.com/windmill-labs/windmill/issues/11100)) ([e877b5f](https://github.com/windmill-labs/windmill/commit/e877b5f2e81b1741aee90e843c8cccc22f9eef24)) +* stop a resource delete from taking variables it does not own ([#11102](https://github.com/windmill-labs/windmill/issues/11102)) ([2a21efa](https://github.com/windmill-labs/windmill/commit/2a21efa11b8307b331a8c20720028444dd3c62ff)) + +## [1.810.0](https://github.com/windmill-labs/windmill/compare/v1.809.0...v1.810.0) (2026-09-11) + + +### Features + +* **ai-sessions:** turn skills on by default, and group them by folder ([#11058](https://github.com/windmill-labs/windmill/issues/11058)) ([d8b9174](https://github.com/windmill-labs/windmill/commit/d8b9174235b97d0b4ef9f281e20e5704182d8dcb)) +* background and wait_seconds for run_script, skip preprocessor ([#11092](https://github.com/windmill-labs/windmill/issues/11092)) ([2939c2d](https://github.com/windmill-labs/windmill/commit/2939c2dd4b129640d87ef45c7a24c6f215a3239a)) +* give the chat the full MCP tool schema, and mark calls with the provider icon ([#11086](https://github.com/windmill-labs/windmill/issues/11086)) ([e7c6f85](https://github.com/windmill-labs/windmill/commit/e7c6f85553bd8efb0f7af0f488f615ac93c496ae)) +* let apps hide the viewer login status on public urls ([#11089](https://github.com/windmill-labs/windmill/issues/11089)) ([6056ec7](https://github.com/windmill-labs/windmill/commit/6056ec7148bce9f8ed171dd29f544696c335de7d)) +* remove the viewer login status badge from public apps ([#11090](https://github.com/windmill-labs/windmill/issues/11090)) ([75d7bee](https://github.com/windmill-labs/windmill/commit/75d7bee178886461fe49090d606a708f25c02d1a)) +* run a deployed flow through the chat's argument form ([#11085](https://github.com/windmill-labs/windmill/issues/11085)) ([b50de89](https://github.com/windmill-labs/windmill/commit/b50de8947908f1a5a4e9472afe6c0ecd25892e93)) +* run a flow test through the chat's argument form ([#11069](https://github.com/windmill-labs/windmill/issues/11069)) ([172d6c2](https://github.com/windmill-labs/windmill/commit/172d6c275b92b39d13848b543df9db10148e94ef)) + + +### Bug Fixes + +* attach TLS to gRPC OTLP exporters for https endpoints ([#11078](https://github.com/windmill-labs/windmill/issues/11078)) ([f915ed6](https://github.com/windmill-labs/windmill/commit/f915ed6a46e14341ddb213e869090edb68763032)) +* **dbt:** stop dbt sending anonymous usage stats from workers ([#11091](https://github.com/windmill-labs/windmill/issues/11091)) ([d539e86](https://github.com/windmill-labs/windmill/commit/d539e8674f2bf92d6c10b182ed4a1073714062c0)) +* keep pinned import versions of imported scripts in bun lockfiles ([#11082](https://github.com/windmill-labs/windmill/issues/11082)) ([57f8b08](https://github.com/windmill-labs/windmill/commit/57f8b0826ad61cb118d0cafbbc9203327d858940)) +* let admins and background sync reach private git hosts ([#11084](https://github.com/windmill-labs/windmill/issues/11084)) ([fa53099](https://github.com/windmill-labs/windmill/commit/fa53099e2b87a8676c5d6a18e77844e17ff45efd)) +* serve instance env settings at the documented /settings/local path ([#11075](https://github.com/windmill-labs/windmill/issues/11075)) ([f8f7c00](https://github.com/windmill-labs/windmill/commit/f8f7c0009f32c1440566420725b12e78ca804b03)) +* show symlinked files in the git repo viewer ([#11081](https://github.com/windmill-labs/windmill/issues/11081)) ([e6d4f44](https://github.com/windmill-labs/windmill/commit/e6d4f44a6122dab47fbee2a2a2b4330a62841bed)) +* support gzip and zstd compression for OTLP export over gRPC ([#11077](https://github.com/windmill-labs/windmill/issues/11077)) ([b156778](https://github.com/windmill-labs/windmill/commit/b156778da24e3827f723e503f80df68de87ddf7e)) +* unpin only the specifiers in the bundle a bun modules run executes ([#11083](https://github.com/windmill-labs/windmill/issues/11083)) ([30ffdbe](https://github.com/windmill-labs/windmill/commit/30ffdbecc15270562ceed3030c50a1cf81b1195c)) + + +### Performance Improvements + +* lazy-load the low-code runtime on public app pages ([#11087](https://github.com/windmill-labs/windmill/issues/11087)) ([e651b4c](https://github.com/windmill-labs/windmill/commit/e651b4cd63c3bd64a8739c9f60c6dfa437f19b4b)) + +## [1.809.0](https://github.com/windmill-labs/windmill/compare/v1.808.0...v1.809.0) (2026-09-10) + + +### Features + +* add a minimal skin for the approval page and slack/teams ([#11061](https://github.com/windmill-labs/windmill/issues/11061)) ([63cb46d](https://github.com/windmill-labs/windmill/commit/63cb46d7bb9db1d996aa38e06bd3afebc60111bd)) +* live queue status per tag and bounded queue metric charts ([#11067](https://github.com/windmill-labs/windmill/issues/11067)) ([569adb8](https://github.com/windmill-labs/windmill/commit/569adb85c1885d289e80a70a166f0f74e6d5ba83)) +* **otel:** read the OTLP metrics temporality preference ([#11064](https://github.com/windmill-labs/windmill/issues/11064)) ([2f88769](https://github.com/windmill-labs/windmill/commit/2f8876908719b3640d7cfc9364a7b0f7145fc356)) +* **otel:** support standard OTEL resource attribute env vars ([#10974](https://github.com/windmill-labs/windmill/issues/10974)) ([0a40eea](https://github.com/windmill-labs/windmill/commit/0a40eea37a7dbde5fc4760d6333d81186dfee255)) +* report script metadata with no content file in wmill lint ([#11053](https://github.com/windmill-labs/windmill/issues/11053)) ([8820b9f](https://github.com/windmill-labs/windmill/commit/8820b9fc644c6620c50517cd0d902015e2e670e2)) +* show the workspace an operator is in, and let them switch ([#11059](https://github.com/windmill-labs/windmill/issues/11059)) ([385086f](https://github.com/windmill-labs/windmill/commit/385086ffc21c72cd07624584932a4a301c23a732)) +* tuck other users' spaces into a collapsible home tree row ([#11073](https://github.com/windmill-labs/windmill/issues/11073)) ([d87f089](https://github.com/windmill-labs/windmill/commit/d87f089288996af9ea7e3b017a5ef35d4ded880d)) + + +### Bug Fixes + +* **ai-chat:** test_run_flow could test a different flow than the one asked ([#11066](https://github.com/windmill-labs/windmill/issues/11066)) ([fa73539](https://github.com/windmill-labs/windmill/commit/fa73539839071491fb2cbad9242f52ad22b975bf)) +* bound list_jobs runtime and paginate runs on the sorted column ([#11072](https://github.com/windmill-labs/windmill/issues/11072)) ([f517402](https://github.com/windmill-labs/windmill/commit/f51740253871960b55ab2aa8989e8df3fbde0351)) +* **frontend:** clear the flow graph selection through xyflow's store ([#11056](https://github.com/windmill-labs/windmill/issues/11056)) ([b4be8bc](https://github.com/windmill-labs/windmill/commit/b4be8bc5354fbd3a47c267c331aaf603c0f90e6e)) +* **frontend:** recompute dataflow edges when selecting a step ([#11070](https://github.com/windmill-labs/windmill/issues/11070)) ([08d876a](https://github.com/windmill-labs/windmill/commit/08d876aebf32ebb995a8c1839aa87794c0176bed)) +* **frontend:** restore heading sizes in note markdown and keep group notes on id change ([#11047](https://github.com/windmill-labs/windmill/issues/11047)) ([e63072c](https://github.com/windmill-labs/windmill/commit/e63072c216383700a23504be89782f4c69657174)) +* give every table a primary key so the db can be logically replicated ([#11036](https://github.com/windmill-labs/windmill/issues/11036)) ([e62bfdc](https://github.com/windmill-labs/windmill/commit/e62bfdcd8c6f1389601ccd2b5809c5eff0ff262e)) +* keep an app's deployed policy on wmill push ([#11049](https://github.com/windmill-labs/windmill/issues/11049)) ([0af7675](https://github.com/windmill-labs/windmill/commit/0af7675588300863883a99c7f14bc53cafec8a7e)) +* refuse cross-site GET requests that run Hub scripts ([#11054](https://github.com/windmill-labs/windmill/issues/11054)) ([ab9efc8](https://github.com/windmill-labs/windmill/commit/ab9efc897cc94d61a67263772806b15e9225cece)) +* skip the deploy PR when the git sync push committed nothing ([#11076](https://github.com/windmill-labs/windmill/issues/11076)) ([8ecbd33](https://github.com/windmill-labs/windmill/commit/8ecbd339eef7314a93d599fcea4377299d4c493d)) +* space the trailing AI settings cards ([#11044](https://github.com/windmill-labs/windmill/issues/11044)) ([5d7eed1](https://github.com/windmill-labs/windmill/commit/5d7eed1c02b0966289cc8cca00a15d76dad187a5)) +* stop uv pip compile emitting lockfile annotations ([#11042](https://github.com/windmill-labs/windmill/issues/11042)) ([8aa8b7e](https://github.com/windmill-labs/windmill/commit/8aa8b7ee6c23f859f169637c0bfd3f8d964509e9)) +* surface why a private or untrusted git host is unreachable ([#11068](https://github.com/windmill-labs/windmill/issues/11068)) ([c57b18e](https://github.com/windmill-labs/windmill/commit/c57b18e46fcdd319213fe0a537cb18de418ca688)) + + +### Performance Improvements + +* index the FK columns that cascade on workspace delete ([#11052](https://github.com/windmill-labs/windmill/issues/11052)) ([9a563f6](https://github.com/windmill-labs/windmill/commit/9a563f6d72da28fe09b785cd0683e9698df72bba)) +* only write queue metrics when a tag's backlog changes ([#11055](https://github.com/windmill-labs/windmill/issues/11055)) ([9d75929](https://github.com/windmill-labs/windmill/commit/9d75929247ea2ec39286971fcbebf95d886194f3)) + +## [1.808.0](https://github.com/windmill-labs/windmill/compare/v1.807.0...v1.808.0) (2026-09-09) + + +### Features + +* run and test scripts from the AI chat through an argument form ([#11001](https://github.com/windmill-labs/windmill/issues/11001)) ([a6abf2c](https://github.com/windmill-labs/windmill/commit/a6abf2c8a744e9ee6acf4830cdfbb84f2f95cb36)) + +## [1.807.0](https://github.com/windmill-labs/windmill/compare/v1.806.0...v1.807.0) (2026-09-09) + + +### Features + +* add a dismissible instance-wide announcement banner ([#11037](https://github.com/windmill-labs/windmill/issues/11037)) ([abf4c6c](https://github.com/windmill-labs/windmill/commit/abf4c6c2348014ea4401b9be62b5b15e5800e879)) +* batch chained DDL statements into a single migration ([#11038](https://github.com/windmill-labs/windmill/issues/11038)) ([656e609](https://github.com/windmill-labs/windmill/commit/656e609595833bc854f845d35cf43157e76d732f)) +* create the cloud workspace in onboarding, and teach the empty home ([#10959](https://github.com/windmill-labs/windmill/issues/10959)) ([fd35b47](https://github.com/windmill-labs/windmill/commit/fd35b4765843879cb2254f402c142fd7510f1916)) +* link from the public run view to the authenticated run page ([#11041](https://github.com/windmill-labs/windmill/issues/11041)) ([09b81a9](https://github.com/windmill-labs/windmill/commit/09b81a9294bed2795a1a7b688d0b02e0a61957ea)) +* make guest access unavailable on the shared cloud ([#11040](https://github.com/windmill-labs/windmill/issues/11040)) ([0b63e0a](https://github.com/windmill-labs/windmill/commit/0b63e0a6929088ff25def4fd6547cf61668251a5)) + + +### Bug Fixes + +* ignore comments and continuations in python lockfiles ([#11035](https://github.com/windmill-labs/windmill/issues/11035)) ([90c4e10](https://github.com/windmill-labs/windmill/commit/90c4e1020a2ff896977648dd68b572413cea7709)) +* refetch an unparseable hub script cache entry instead of panicking ([#11033](https://github.com/windmill-labs/windmill/issues/11033)) ([88c3ebd](https://github.com/windmill-labs/windmill/commit/88c3ebdfc1325ffbea4521d854e71d231a6409c4)) +* stop a new AI session adopting a legacy sidebar chat ([#11039](https://github.com/windmill-labs/windmill/issues/11039)) ([1076b63](https://github.com/windmill-labs/windmill/commit/1076b638d987ba99c5c27e5478ca580534b9d572)) + +## [1.806.0](https://github.com/windmill-labs/windmill/compare/v1.805.0...v1.806.0) (2026-09-08) + + +### Features + +* bring gitlab repositories to parity for git sync ([#10938](https://github.com/windmill-labs/windmill/issues/10938)) ([9444049](https://github.com/windmill-labs/windmill/commit/9444049d6013c77a5f25f01a736eb5cd741fb3e6)) +* draw a dbt column trace, across projects and the pipeline boundary ([#11014](https://github.com/windmill-labs/windmill/issues/11014)) ([33f9828](https://github.com/windmill-labs/windmill/commit/33f9828c3ed15fe63fccedc1550584f15c0490ab)) +* durable dbt state per environment, and `--defer` onto it ([#10975](https://github.com/windmill-labs/windmill/issues/10975)) ([621fac5](https://github.com/windmill-labs/windmill/commit/621fac55abcd1859e8c8c06e5f4412e61bb85d59)) +* ingest dbt column lineage and real column schemas from the engine's parquet index ([#10977](https://github.com/windmill-labs/windmill/issues/10977)) ([0139467](https://github.com/windmill-labs/windmill/commit/0139467b01b82e4b3d474ca3f205358fa607d19a)) +* let a worker group override the dependency cache object store ([#11019](https://github.com/windmill-labs/windmill/issues/11019)) ([de98adf](https://github.com/windmill-labs/windmill/commit/de98adf055835ab7c4d6305e7d5d3bdab915876b)) +* **nativets:** bound fetch on a peer that never answers ([#11026](https://github.com/windmill-labs/windmill/issues/11026)) ([785277e](https://github.com/windmill-labs/windmill/commit/785277e0bb2ea77b71a89dd4d389d439dfcf9e03)) +* recognize `// volume:` mounts in PHP scripts ([#11018](https://github.com/windmill-labs/windmill/issues/11018)) ([f081fb1](https://github.com/windmill-labs/windmill/commit/f081fb10705cadf99e99dfa786d1cc2ebf0447db)) +* report a WAC task failure the workflow body never awaited ([#11017](https://github.com/windmill-labs/windmill/issues/11017)) ([3e3a41d](https://github.com/windmill-labs/windmill/commit/3e3a41d418d4ee3fe060bd3acf3324f311d89c8a)) +* retry a workflow-as-code task from its task options ([#11013](https://github.com/windmill-labs/windmill/issues/11013)) ([d3f305d](https://github.com/windmill-labs/windmill/commit/d3f305db982b7c5dc49babf9bec8b62adcd2557d)) + + +### Bug Fixes + +* chain redeploys onto a retired path's version history ([#11029](https://github.com/windmill-labs/windmill/issues/11029)) ([0b37226](https://github.com/windmill-labs/windmill/commit/0b372260787edb7e9627ad4fb6637ad5f1024e0a)) +* make the native trigger disable/enable toggle actually save ([#11024](https://github.com/windmill-labs/windmill/issues/11024)) ([448fce9](https://github.com/windmill-labs/windmill/commit/448fce93f743d5b2ef2a2d4496eb2ec238594a0a)) +* offload php signature parsing from async workers ([#11027](https://github.com/windmill-labs/windmill/issues/11027)) ([2cb02e3](https://github.com/windmill-labs/windmill/commit/2cb02e3b3398db49f16377dd79dde2dd6fb5cc02)) +* reduce php parser stack use in debug workers ([#11025](https://github.com/windmill-labs/windmill/issues/11025)) ([2ae8509](https://github.com/windmill-labs/windmill/commit/2ae8509b14f112c9ef5b71321e8fd52596a16c10)) + + +### Performance Improvements + +* reduce shared worker debug polling frames ([#11028](https://github.com/windmill-labs/windmill/issues/11028)) ([946756a](https://github.com/windmill-labs/windmill/commit/946756ae83deb4e7a93111edddbd4d98c596d5a9)) + +## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) + + +### Features + +* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9)) +* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63)) +* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c)) +* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4)) +* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a)) +* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab)) + + +### Bug Fixes + +* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728)) +* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198)) +* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43)) +* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c)) +* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac)) +* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de)) +* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f)) +* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f)) +* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13)) + ## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05) diff --git a/Dockerfile b/Dockerfile index 6a290c3553..0fec95b95a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,6 +73,8 @@ COPY /backend/oauth_connect.json /backend/oauth_connect.json COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh COPY /system_prompts/auto-generated /system_prompts/auto-generated +# The flow chat imports the chat SDK's source (svelte.config.js alias `windmill-chat`). +COPY /chat-sdk/src /chat-sdk/src RUN cd /backend/windmill-api && . ./build_openapi.sh COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 4501f24810..2e29b9064d 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -132,6 +132,10 @@ export async function runEval( setToolStatus: () => {}, removeToolStatus: () => {}, isPlanModeActive, + // Accepts the run form exactly as the model prefilled it: there is nobody here to + // edit the arguments, so a case can assert what the model proposed but never how + // it reacts to the user changing something. + requestRunArgs: async (_toolId, form) => form.args, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 338ed8504c..e5b275b86e 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -86,6 +86,7 @@ vi.mock('$lib/gen', async () => { previewBenchmarkSchedule, runBenchmarkDatatableSql, runBenchmarkFlowByPath, + runBenchmarkScriptByPath, runBenchmarkScriptPreview, updateBenchmarkDraft, listBenchmarkMcpTools @@ -279,6 +280,18 @@ vi.mock('$lib/gen', async () => { } return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody }) }, + runScriptByPath: async (data: { + workspace: string + path: string + requestBody?: Record + }) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkScriptByPath({ + workspace: data.workspace, + path: data.path, + args: data.requestBody + }) + : actual.JobService.runScriptByPath(data), runFlowByPath: async (data: { workspace: string path: string diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 94ecb5c029..71693156a9 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1974,6 +1974,144 @@ judgeChecklist: - deletes the deployed script via delete_workspace_item rather than a raw API endpoint +- id: global-test33-run-deployed-script-with-form + prompt: |- + Run the deployed script `f/evals/global/format_greeting` for me with the name "ada". + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_script + - call_api_endpoint + - write_script + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_script + field: args.name + stringIncludesAnyOf: + - ada + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed script through run_script rather than a preview test run or a raw API endpoint + - passes the name "ada" so the confirmation form comes up prefilled + +- id: global-test34-run-with-secret-from-variable + prompt: |- + Run the deployed `f/evals/global/billing_sync` for the account `acme` — use the billing + API token we already keep in the workspace. + initial: ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json + runtime: + maxTurns: 10 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + # A secret argument is filled by naming the variable that holds it: the value stays in + # the variable and only its path travels. A literal reaches the job as a reference too, + # minted on the way in, but it stays in the tool call the model emitted. + toolCallArgs: + - tool: run_script + field: args.api_token + stringIncludesAnyOf: + - "$var:f/evals/global/stripe_api_token" + - tool: run_script + field: args.account + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - fills the secret argument with a reference to the existing workspace variable rather than a literal token + - passes the account "acme" + - does not invent or guess the token's value + +- id: global-test35-run-deployed-flow-with-form + prompt: |- + Run the deployed flow `f/evals/global/notify_customer` for me — the customer is `acme`. + initial: ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_flow + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_flow + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_flow + - call_api_endpoint + - write_flow + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_flow + field: args.customer + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed flow through run_flow rather than a preview test run or a raw API endpoint + - passes the customer "acme" so the confirmation form comes up prefilled + +- id: global-test36-draft-flow-test-run-not-deployed + prompt: |- + Update the `calculate_total` step of `f/evals/global/process_invoice` so it applies 8% tax and + returns `subtotal`, `tax` and `total`, then run it to check it works. + Keep it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + path: f/evals/global/process_invoice + toolExpect: + # A one-step flow is as well checked by running the step as the whole flow, so both + # count: what matters is that the run is against the draft. + requiredToolsAnyOf: + - [test_run_flow, test_run_step] + # The draft is what the user asked to check, and run_flow would run the deployed + # version instead — the edit would not be in what ran. + forbiddenToolsUsed: + - run_flow + - call_api_endpoint + - deploy_workspace_item + # The judge cannot observe runs, and the edit's content is already pinned by + # global-test5 on this fixture; what this case guards is where the run went. + skipJudge: true + judgeChecklist: + - creates an AI draft of f/evals/global/process_invoice applying 8% tax + - does not deploy or save the draft + - id: global-undo-created-draft prompt: |- Create a draft Postgres resource at `u/admin/scratch_db` for host db.example.com port 5432, database `orders`, user `app`, and tell me what fields it ended up with. diff --git a/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json new file mode 100644 index 0000000000..ef166b0f9c --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "variables": [ + { + "path": "f/evals/global/stripe_api_token", + "value": "sk_live_do_not_leak_me", + "is_secret": true, + "description": "Token used by the billing sync job", + "labels": ["billing"] + } + ], + "scripts": [ + { + "path": "f/evals/global/billing_sync", + "summary": "Sync billing records", + "description": "Syncs billing records for one account, authenticating with an API token.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "account": { + "type": "string" + }, + "api_token": { + "type": "string", + "password": true, + "description": "API token to authenticate with" + } + }, + "required": ["account", "api_token"] + }, + "content": "export async function main(account: string, api_token: string) {\n return `synced ${account}`\n}\n" + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json new file mode 100644 index 0000000000..d8cbf37ace --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json @@ -0,0 +1,40 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/notify_customer", + "summary": "Notify a customer", + "description": "Sends a notification to the named customer.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "customer": { + "type": "string" + } + }, + "required": ["customer"] + }, + "value": { + "modules": [ + { + "id": "notify", + "summary": "Send the notification", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(customer: string) {\n return `Notified ${customer}`\n}\n", + "input_transforms": { + "customer": { + "type": "javascript", + "expr": "flow_input.customer" + } + } + } + } + ] + } + } + ] + } +} diff --git a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json b/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json deleted file mode 100644 index 9c1f8d00a7..0000000000 --- a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((\n SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM v2_job_queue\n WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1\n )))", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f" -} diff --git a/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json new file mode 100644 index 0000000000..6558ea16e5 --- /dev/null +++ b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_shared_artifact\n (workspace_id, artifact_id, email, created_by, name, kind, version, content)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE\n SET created_by = EXCLUDED.created_by,\n name = EXCLUDED.name,\n kind = EXCLUDED.kind,\n version = EXCLUDED.version,\n content = EXCLUDED.content,\n shared_at = now()\n RETURNING id, shared_at, (xmax = 0) AS \"inserted!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "shared_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "inserted!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int4", + "Text" + ] + }, + "nullable": [ + false, + false, + null + ] + }, + "hash": "0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a" +} diff --git a/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json new file mode 100644 index 0000000000..db9d674fcf --- /dev/null +++ b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock',\n 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily',\n 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d" +} diff --git a/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json b/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json new file mode 100644 index 0000000000..0aad412ebe --- /dev/null +++ b/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO login_link (token_hash, email, rd, expiration, created_by)\n VALUES ($1, $2, $3, $4, $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bpchar", + "Varchar", + "Text", + "Timestamptz", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02" +} diff --git a/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json b/backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json similarity index 83% rename from backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json rename to backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json index 1c0035f7cb..8aecbed333 100644 --- a/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json +++ b/backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary,\n enabled\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -100,8 +105,9 @@ true, false, false, - true + true, + false ] }, - "hash": "6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa" + "hash": "0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd" } diff --git a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json similarity index 78% rename from backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json rename to backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json index f950c5bf57..da25d2fcf8 100644 --- a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json +++ b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236" + "hash": "0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63" } diff --git a/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json new file mode 100644 index 0000000000..446c785770 --- /dev/null +++ b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT u.username, u.email FROM usr u WHERE u.workspace_id = $1 AND u.is_admin AND NOT u.operator AND NOT u.disabled AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) ORDER BY u.username LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388" +} diff --git a/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json new file mode 100644 index 0000000000..00b839de6a --- /dev/null +++ b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf" +} diff --git a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json similarity index 76% rename from backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json rename to backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json index 37115e4ab7..780fb75ea9 100644 --- a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json +++ b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314" + "hash": "139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08" } diff --git a/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json b/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json new file mode 100644 index 0000000000..f5bb8e14d1 --- /dev/null +++ b/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO git_sync_ci_test_check\n (workspace_id, poster_workspace_id, head_sha, head_ref, repo_url,\n repo_resource_path, check_run_id,\n created_at, concluded, conclusion, concluded_at, github_posted)\n VALUES ($1, $2, $3, $4, $5, $6, NULL, now(), false, NULL, NULL, false)\n ON CONFLICT (workspace_id, repo_resource_path, head_sha) DO UPDATE SET\n poster_workspace_id = EXCLUDED.poster_workspace_id,\n head_ref = EXCLUDED.head_ref,\n repo_url = EXCLUDED.repo_url,\n check_run_id = NULL,\n created_at = now(),\n concluded = false,\n conclusion = NULL,\n concluded_at = NULL,\n github_posted = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6" +} diff --git a/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json new file mode 100644 index 0000000000..7bb2cac5db --- /dev/null +++ b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6" +} diff --git a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json b/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json deleted file mode 100644 index bc2e6a6f63..0000000000 --- a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COALESCE(username, split_part(email, '@', 1)) AS \"username!\", email FROM password WHERE super_admin = true AND disabled = false ORDER BY email LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "username!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - false - ] - }, - "hash": "17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3" -} diff --git a/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json b/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json new file mode 100644 index 0000000000..8647442530 --- /dev/null +++ b/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT test_script_path, tested_item_path, tested_item_kind, has_wildcard AS \"has_wildcard!\" FROM ci_test_reference WHERE workspace_id = $1 ORDER BY test_script_path, tested_item_kind, tested_item_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "test_script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "tested_item_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "tested_item_kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "has_wildcard!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9" +} diff --git a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json b/backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json similarity index 61% rename from backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json rename to backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json index 0a833ba620..a78e343072 100644 --- a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json +++ b/backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT username, email FROM usr WHERE workspace_id = $1 AND is_admin = true AND operator = false AND disabled = false ORDER BY username LIMIT 1", + "query": "SELECT u.username, u.email FROM workspace w JOIN usr u ON u.workspace_id = w.id AND u.email = w.owner WHERE w.id = $1 AND NOT u.disabled", "describe": { "columns": [ { @@ -24,5 +24,5 @@ false ] }, - "hash": "3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c" + "hash": "1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7" } diff --git a/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json b/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json new file mode 100644 index 0000000000..b9cbc6c382 --- /dev/null +++ b/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d" +} diff --git a/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json new file mode 100644 index 0000000000..b60dd43fdf --- /dev/null +++ b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6" +} diff --git a/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json new file mode 100644 index 0000000000..77b00d9daa --- /dev/null +++ b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704" +} diff --git a/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json new file mode 100644 index 0000000000..6b4324af4c --- /dev/null +++ b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3" +} diff --git a/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json new file mode 100644 index 0000000000..bdb54cfbb9 --- /dev/null +++ b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c" +} diff --git a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json new file mode 100644 index 0000000000..04b23832d9 --- /dev/null +++ b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Bool", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead" +} diff --git a/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json new file mode 100644 index 0000000000..50755132fc --- /dev/null +++ b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4,\n 'source', $4, $5, '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6,\n 'model', $6, $7, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c" +} diff --git a/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json b/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json new file mode 100644 index 0000000000..29ec70e75e --- /dev/null +++ b/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330" +} diff --git a/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json new file mode 100644 index 0000000000..5f3d8300eb --- /dev/null +++ b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH slots AS (\n SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak,\n (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value,\n (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing,\n COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp,\n max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb\n FROM (\n SELECT id, t,\n CASE jsonb_typeof(value)\n WHEN 'number' THEN value::double precision\n WHEN 'object' THEN t - (value->>'since')::double precision\n END AS v,\n (value->>'since')::double precision AS since,\n (jsonb_typeof(value) = 'object')::int::double precision AS climbing,\n greatest(floor((t - $1::double precision) / $2::double precision), -1)::int\n AS slot\n FROM (\n SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > to_timestamp($1::double precision - $3::double precision)\n ) m\n ) s\n WHERE v IS NOT NULL\n GROUP BY id, slot\n )\n SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\",\n greatest(peak, CASE WHEN last_climb[1] < last THEN (\n SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision\n FROM metrics n\n WHERE n.id = slots.id AND n.id LIKE 'queue_%'\n AND n.created_at > to_timestamp(last_climb[1] + 0.001)\n AND n.created_at <= to_timestamp(last + 0.001)\n ) - last_climb[2] END) AS \"peak!\",\n first_value AS \"first_value!\", last_value AS \"last_value!\",\n last_climbing AS \"last_climbing!\", ramp AS \"ramp!\"\n FROM slots\n ORDER BY id, slot", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slot!", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "first!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "last!", + "type_info": "Float8" + }, + { + "ordinal": 4, + "name": "peak!", + "type_info": "Float8" + }, + { + "ordinal": 5, + "name": "first_value!", + "type_info": "Float8" + }, + { + "ordinal": 6, + "name": "last_value!", + "type_info": "Float8" + }, + { + "ordinal": 7, + "name": "last_climbing!", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "ramp!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Float8", + "Float8", + "Float8", + "Float8" + ] + }, + "nullable": [ + false, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863" +} diff --git a/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json b/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json deleted file mode 100644 index e4f70e1f07..0000000000 --- a/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_completed SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3" -} diff --git a/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json new file mode 100644 index 0000000000..919d930f70 --- /dev/null +++ b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887" +} diff --git a/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json b/backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json similarity index 65% rename from backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json rename to backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json index 697e49ab9d..3a864fee63 100644 --- a/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json +++ b/backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json @@ -1,15 +1,23 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + "query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb\n RETURNING duration_ms AS \"duration_ms!\"", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "duration_ms!", + "type_info": "Int8" + } + ], "parameters": { "Left": [ "Uuid", "Jsonb" ] }, - "nullable": [] + "nullable": [ + false + ] }, - "hash": "beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb" + "hash": "2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f" } diff --git a/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json new file mode 100644 index 0000000000..7148252002 --- /dev/null +++ b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188" +} diff --git a/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json b/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json new file mode 100644 index 0000000000..99c8262860 --- /dev/null +++ b/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n route_path = $1,\n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n allowed_origins = $6,\n authentication_resource_path = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n mode = $11,\n http_method = $12,\n static_asset_config = $13,\n edited_by = $14,\n permissioned_as = $15,\n request_type = $16,\n authentication_method = $17,\n summary = $18,\n description = $19,\n edited_at = now(),\n is_static_website = $20,\n error_handler_path = $21,\n error_handler_args = $22,\n retry = $23\n WHERE\n workspace_id = $24 AND\n path = $25\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908" +} diff --git a/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json b/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json new file mode 100644 index 0000000000..54da0105e7 --- /dev/null +++ b/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE password SET password_hash = $1, login_type = 'password'\n WHERE email = $2 AND login_type = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d" +} diff --git a/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json new file mode 100644 index 0000000000..0937a312d1 --- /dev/null +++ b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094" +} diff --git a/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json b/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json new file mode 100644 index 0000000000..941dc122d0 --- /dev/null +++ b/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f" +} diff --git a/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json b/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json new file mode 100644 index 0000000000..0a2976f868 --- /dev/null +++ b/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "flow_step_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9" +} diff --git a/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json new file mode 100644 index 0000000000..fbfab14017 --- /dev/null +++ b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db" +} diff --git a/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json b/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json new file mode 100644 index 0000000000..9f9cf036fb --- /dev/null +++ b/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r->>'script_path' as \"script_path\"\n FROM workspace_settings ws,\n jsonb_array_elements(\n CASE WHEN jsonb_typeof(ws.git_sync->'repositories') = 'array'\n THEN ws.git_sync->'repositories' END\n ) r\n WHERE ws.workspace_id = $1\n AND r->>'git_repo_resource_path' IN ($2, '$res:' || $2)\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1" +} diff --git a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json similarity index 76% rename from backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json rename to backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json index d2084d76d8..76e897d3a4 100644 --- a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json +++ b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -58,8 +63,9 @@ true, true, true, - false + false, + true ] }, - "hash": "623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4" + "hash": "36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907" } diff --git a/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json b/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json new file mode 100644 index 0000000000..ebaaf0b755 --- /dev/null +++ b/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of_email' = $2 AND (value->'policy'->>'on_behalf_of' IS NULL OR value->'policy'->>'on_behalf_of' NOT LIKE 'g/%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da" +} diff --git a/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json similarity index 70% rename from backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json rename to backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json index 3b9a2e2f34..2f794e6be0 100644 --- a/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json +++ b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", + "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", "describe": { "columns": [ { @@ -15,6 +15,8 @@ "Jsonb", "Text", "Text", + "Bool", + "Text", "Bool" ] }, @@ -22,5 +24,5 @@ null ] }, - "hash": "8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a" + "hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34" } diff --git a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json similarity index 75% rename from backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json rename to backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json index ffd1670c04..87d740a247 100644 --- a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json +++ b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6" + "hash": "38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a" } diff --git a/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json new file mode 100644 index 0000000000..28b34afa97 --- /dev/null +++ b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind,\n ingested_at)\n SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column,\n child_unique_id, child_column, lineage_kind, ingested_at\n FROM dbt_column_edge\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4" +} diff --git a/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json new file mode 100644 index 0000000000..843484dd86 --- /dev/null +++ b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "now!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753" +} diff --git a/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json new file mode 100644 index 0000000000..1268656e34 --- /dev/null +++ b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest, manifest_key, run_results,\n run_results_key, updated_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET\n job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest,\n manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results,\n run_results_key = EXCLUDED.run_results_key, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Uuid", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3" +} diff --git a/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json new file mode 100644 index 0000000000..564c784d22 --- /dev/null +++ b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin AS \"super_admin!\" FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c" +} diff --git a/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json b/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json new file mode 100644 index 0000000000..2d0bb40c18 --- /dev/null +++ b/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET concluded = true, github_posted = true, concluded_at = now(),\n conclusion = COALESCE(conclusion, 'failure')\n WHERE (check_run_id IS NULL AND NOT concluded\n AND created_at < now() - make_interval(secs => $1))\n OR (concluded AND NOT github_posted\n AND concluded_at < now() - make_interval(secs => $2))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586" +} diff --git a/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json b/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json new file mode 100644 index 0000000000..5522f3bb17 --- /dev/null +++ b/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT profile FROM cloud_onboarding_profile WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "profile", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743" +} diff --git a/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json new file mode 100644 index 0000000000..7f950346ee --- /dev/null +++ b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4" +} diff --git a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json b/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json deleted file mode 100644 index 9f97b97a88..0000000000 --- a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH queue_metrics as (\n SELECT id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "values!", - "type_info": "JsonArray" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - null - ] - }, - "hash": "44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa" -} diff --git a/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json b/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json new file mode 100644 index 0000000000..526f3f5537 --- /dev/null +++ b/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59" +} diff --git a/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json new file mode 100644 index 0000000000..9b361b8f53 --- /dev/null +++ b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru\n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf" +} diff --git a/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json b/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json new file mode 100644 index 0000000000..6bec80ccb2 --- /dev/null +++ b/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json @@ -0,0 +1,84 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n wrap_body = $1,\n raw_string = $2,\n allowed_origins = $3,\n authentication_resource_path = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n mode = $8,\n http_method = $9,\n static_asset_config = $10,\n edited_by = $11,\n permissioned_as = $12,\n request_type = $13,\n authentication_method = $14,\n summary = $15,\n description = $16,\n edited_at = now(),\n is_static_website = $17,\n error_handler_path = $18,\n error_handler_args = $19,\n retry = $20\n WHERE\n workspace_id = $21 AND\n path = $22\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc" +} diff --git a/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json new file mode 100644 index 0000000000..b2cfb1cd89 --- /dev/null +++ b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem->'credential'\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') AS elem\n WHERE workspace_id = $1 AND elem->>'git_repo_resource_path' IN ($2, $3)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6" +} diff --git a/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json new file mode 100644 index 0000000000..24e563d9ad --- /dev/null +++ b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "edge!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e" +} diff --git a/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json new file mode 100644 index 0000000000..913c16c31d --- /dev/null +++ b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text\n WHERE EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false\n AND language = 'dbt'\n AND (hash = $4 OR $4 = ANY(parent_hashes)))\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0" +} diff --git a/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json b/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json new file mode 100644 index 0000000000..8c784b664e --- /dev/null +++ b/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = now()\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4\n AND (tests_dispatched_at IS NULL\n OR (ci_test_job_ids IS NULL\n AND tests_dispatched_at < now() - make_interval(secs => $5)))\n RETURNING true as \"claimed!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "claimed!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da" +} diff --git a/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json b/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json new file mode 100644 index 0000000000..c8b8b94da4 --- /dev/null +++ b/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (\n SELECT h.sha FROM git_sync_synced_head h\n WHERE h.workspace_id = $1 AND h.repo_resource_path = $4 AND h.branch = $3\n ORDER BY h.synced_at DESC LIMIT 1\n ) = $2 AND NOT EXISTS (\n SELECT 1\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.kind = 'deploymentcallback'\n AND j.args->'__git_sync_auto_pull'->>'branch' = $3\n AND j.args->'__git_sync_auto_pull'->>'repo_resource_path'\n IN ($4, '$res:' || $4)\n ) AND NOT EXISTS (\n SELECT 1\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.kind IN ('dependencies', 'flowdependencies', 'appdependencies')\n ) as \"ready\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ready", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81" +} diff --git a/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json b/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json new file mode 100644 index 0000000000..f7331a55e8 --- /dev/null +++ b/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, company, username, first_time_user)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c" +} diff --git a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json similarity index 70% rename from backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json rename to backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json index da77dc1de5..134e58ed58 100644 --- a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json +++ b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", + "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.column_schema, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", "describe": { "columns": [ { @@ -80,21 +80,26 @@ }, { "ordinal": 15, - "name": "freshness", + "name": "column_schema", "type_info": "Jsonb" }, { "ordinal": 16, + "name": "freshness", + "type_info": "Jsonb" + }, + { + "ordinal": 17, "name": "raw_code", "type_info": "Text" }, { - "ordinal": 17, + "ordinal": 18, "name": "original_file_path", "type_info": "Text" }, { - "ordinal": 18, + "ordinal": 19, "name": "script_visible!", "type_info": "Bool" } @@ -127,8 +132,9 @@ true, true, true, + true, null ] }, - "hash": "9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12" + "hash": "4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5" } diff --git a/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json new file mode 100644 index 0000000000..21759e2fe5 --- /dev/null +++ b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT p.asset_path AS \"from_path!\", e.parent_column AS \"from_column!\",\n c.asset_path AS \"to_path!\", e.child_column AS \"to_column!\",\n e.lineage_kind AS \"kind!\"\n FROM unnest($2::text[], $3::bigint[], $4::uuid[])\n AS o(script_path, script_hash, job_id)\n JOIN dbt_column_edge e ON e.workspace_id = $1\n AND e.script_path = o.script_path\n AND e.job_id = o.job_id\n -- `=` still, with the NULL-to-NULL case\n -- spelled out and gated on the pin: a\n -- version-less row's hash is NULL on both\n -- sides, which `=` never matches, but\n -- `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound everywhere else.\n AND (e.script_hash = o.script_hash\n OR ($5::text IS NOT NULL\n AND o.script_hash IS NULL\n AND e.script_hash IS NULL))\n JOIN dbt_node p ON p.workspace_id = e.workspace_id\n AND p.script_path = e.script_path\n AND p.script_hash IS NOT DISTINCT FROM e.script_hash\n AND p.job_id = e.job_id\n AND p.unique_id = e.parent_unique_id\n JOIN dbt_node c ON c.workspace_id = e.workspace_id\n AND c.script_path = e.script_path\n AND c.script_hash IS NOT DISTINCT FROM e.script_hash\n AND c.job_id = e.job_id\n AND c.unique_id = e.child_unique_id\n WHERE e.lineage_kind IN ('copy', 'mod')\n AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "from_path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "from_column!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "to_path!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "to_column!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array", + "UuidArray", + "Text" + ] + }, + "nullable": [ + true, + false, + true, + false, + false + ] + }, + "hash": "55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d" +} diff --git a/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json new file mode 100644 index 0000000000..822695a6c9 --- /dev/null +++ b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms)\n VALUES ($1, $2, $2, '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d" +} diff --git a/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json new file mode 100644 index 0000000000..b2a0d728cd --- /dev/null +++ b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false AND language = 'dbt'\n AND (hash = $3 OR $3 = ANY(parent_hashes))\n FOR SHARE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147" +} diff --git a/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json new file mode 100644 index 0000000000..f0d9aabea3 --- /dev/null +++ b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\",\n EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\",\n EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\"\n FROM unnest($1::text[]) AS c(id)\n FULL JOIN (\n SELECT DISTINCT ON (id) id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2)\n ORDER BY id, created_at DESC\n ) r ON r.id = c.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "value?", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "at?", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "age?", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Float8" + ] + }, + "nullable": [ + true, + true, + true, + true + ] + }, + "hash": "58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b" +} diff --git a/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json b/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json new file mode 100644 index 0000000000..08008e5900 --- /dev/null +++ b/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin, devops FROM password WHERE email = $1 AND disabled = false FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "devops", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f" +} diff --git a/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json new file mode 100644 index 0000000000..09efdc7340 --- /dev/null +++ b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521" +} diff --git a/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json b/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json new file mode 100644 index 0000000000..c1d11bffe1 --- /dev/null +++ b/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM cloud_trial_offer WHERE email = $1 AND consumed_at IS NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060" +} diff --git a/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json new file mode 100644 index 0000000000..8b04c9e258 --- /dev/null +++ b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10" +} diff --git a/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json b/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json new file mode 100644 index 0000000000..53a3b04d6b --- /dev/null +++ b/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET ci_test_job_ids = NULL, tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c" +} diff --git a/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json b/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json new file mode 100644 index 0000000000..f50b4b743f --- /dev/null +++ b/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific\n WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878" +} diff --git a/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json b/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json new file mode 100644 index 0000000000..1eb8c87315 --- /dev/null +++ b/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO cloud_onboarding_profile (email, profile, created_by) VALUES ($1, $2, $3)\n ON CONFLICT (email) DO UPDATE SET profile = EXCLUDED.profile", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0" +} diff --git a/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json b/backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json similarity index 63% rename from backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json rename to backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json index 89a02933e0..d234750140 100644 --- a/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json +++ b/backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", + "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary,\n enabled\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", "describe": { "columns": [], "parameters": { @@ -23,10 +23,11 @@ "Bool", "Varchar", "Jsonb", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e" + "hash": "64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63" } diff --git a/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json new file mode 100644 index 0000000000..dc1f6ffbb1 --- /dev/null +++ b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2" +} diff --git a/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json new file mode 100644 index 0000000000..8dd7954460 --- /dev/null +++ b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb" +} diff --git a/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json new file mode 100644 index 0000000000..07c923620a --- /dev/null +++ b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7" +} diff --git a/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json new file mode 100644 index 0000000000..b1b92ecc75 --- /dev/null +++ b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "resource_type", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0" +} diff --git a/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json new file mode 100644 index 0000000000..8026b07416 --- /dev/null +++ b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r->'auto_pull'->>'enabled_by' AS \"enabled_by\"\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') r\n WHERE workspace_id = $1 AND r->>'git_repo_resource_path' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled_by", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165" +} diff --git a/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json b/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json new file mode 100644 index 0000000000..427ba00075 --- /dev/null +++ b/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET ci_test_job_ids = $5\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385" +} diff --git a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json similarity index 78% rename from backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json rename to backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json index c44d3d711d..e444c0d549 100644 --- a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json +++ b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -55,8 +60,9 @@ true, true, true, - false + false, + true ] }, - "hash": "e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3" + "hash": "6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251" } diff --git a/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json new file mode 100644 index 0000000000..3a67641732 --- /dev/null +++ b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 6000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837" +} diff --git a/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json new file mode 100644 index 0000000000..7c11dc80c0 --- /dev/null +++ b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232" +} diff --git a/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json b/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json new file mode 100644 index 0000000000..3136e1b433 --- /dev/null +++ b/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE login_link SET consumed_at = now()\n WHERE token_hash = $1 AND consumed_at IS NULL AND expiration > now()\n RETURNING email, rd, created_by", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "rd", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c" +} diff --git a/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json new file mode 100644 index 0000000000..8eac288a1b --- /dev/null +++ b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH running AS (\n SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag\n )\n SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\",\n (SELECT count(*) FROM worker_ping w\n WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag]\n ) AS \"workers!\"\n FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag)\n LEFT JOIN running r ON r.tag = t.tag\n ORDER BY t.tag", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "running!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "workers!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438" +} diff --git a/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json b/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json new file mode 100644 index 0000000000..98aa0d9367 --- /dev/null +++ b/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text))\n WHERE policy->>'on_behalf_of' = $2 AND policy->>'on_behalf_of_email' = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f" +} diff --git a/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json b/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json new file mode 100644 index 0000000000..b7cccbfad4 --- /dev/null +++ b/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check SET github_posted = true\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2\n AND check_run_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51" +} diff --git a/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json new file mode 100644 index 0000000000..ae8cc698b9 --- /dev/null +++ b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag AS \"tag!\", count AS \"count!\",\n EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\",\n EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\"\n FROM (\n SELECT tag, sum(n)::bigint AS count,\n (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head\n FROM (\n SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head\n FROM v2_job_queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag, priority\n ) g\n GROUP BY tag\n ) t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "delay!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "head_since!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null, + null + ] + }, + "hash": "7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5" +} diff --git a/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json new file mode 100644 index 0000000000..c5740cfb88 --- /dev/null +++ b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_environment_state SET script_path = $3\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba" +} diff --git a/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json new file mode 100644 index 0000000000..f4bd63c718 --- /dev/null +++ b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630" +} diff --git a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json similarity index 78% rename from backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json rename to backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json index e4db87ec7d..9f7be9d7cb 100644 --- a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json +++ b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545" + "hash": "82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b" } diff --git a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json b/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json deleted file mode 100644 index f98f3fdeea..0000000000 --- a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92" -} diff --git a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json similarity index 52% rename from backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json rename to backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json index fc354fb9a2..f417ab94f9 100644 --- a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json +++ b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset\n FROM resource_type\n WHERE workspace_id = $1", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name\n FROM resource_type\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986" + "hash": "86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4" } diff --git a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json b/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json deleted file mode 100644 index 124a6e36c5..0000000000 --- a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value) VALUES ($1, $2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15" -} diff --git a/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json new file mode 100644 index 0000000000..a498ecccd0 --- /dev/null +++ b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912" +} diff --git a/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json b/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json new file mode 100644 index 0000000000..e0d369fd12 --- /dev/null +++ b/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET concluded = true, conclusion = $3, concluded_at = now()\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2 AND NOT concluded\n RETURNING check_run_id, poster_workspace_id, repo_url", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "check_run_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "repo_url", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + false, + false + ] + }, + "hash": "89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b" +} diff --git a/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json new file mode 100644 index 0000000000..0bcfd38acd --- /dev/null +++ b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT u.username, u.is_admin, u.operator, u.disabled, EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) AS \"instance_disabled!\" FROM usr u WHERE u.workspace_id = $1 AND u.email = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "disabled", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "instance_disabled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013" +} diff --git a/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json new file mode 100644 index 0000000000..552a38a0d4 --- /dev/null +++ b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_settings.workspace_id AS \"id!\",\n workspace_settings.ai_config->'sessions_retention_days' AS retention\n FROM workspace_settings\n LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2\n WHERE workspace_settings.workspace_id = ANY($1)\n AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "retention", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff" +} diff --git a/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json b/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json new file mode 100644 index 0000000000..db3362bf0c --- /dev/null +++ b/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ci_test_job_ids\n FROM git_sync_synced_head\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ci_test_job_ids", + "type_info": "UuidArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f" +} diff --git a/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json b/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json new file mode 100644 index 0000000000..14a20fa112 --- /dev/null +++ b/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325" +} diff --git a/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json new file mode 100644 index 0000000000..542d523f53 --- /dev/null +++ b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, email, name, kind, version, created_by, content, shared_at\n FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2\n AND shared_at > now() - ($3::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42" +} diff --git a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json deleted file mode 100644 index 823a65cda3..0000000000 --- a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47" -} diff --git a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json deleted file mode 100644 index 3624a32893..0000000000 --- a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at)\n VALUES ('admins', $1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9" -} diff --git a/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json new file mode 100644 index 0000000000..b98a71fb8a --- /dev/null +++ b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26" +} diff --git a/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json b/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json new file mode 100644 index 0000000000..b63a9d294e --- /dev/null +++ b/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039" +} diff --git a/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json b/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json new file mode 100644 index 0000000000..a4a692fd4b --- /dev/null +++ b/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb('u/' || $1))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a" +} diff --git a/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json b/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json new file mode 100644 index 0000000000..9960ba85c9 --- /dev/null +++ b/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd" +} diff --git a/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json new file mode 100644 index 0000000000..6ef7b20a4f --- /dev/null +++ b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e" +} diff --git a/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json new file mode 100644 index 0000000000..49d01c2b8b --- /dev/null +++ b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders',\n 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders',\n 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82" +} diff --git a/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json new file mode 100644 index 0000000000..909e6ad42d --- /dev/null +++ b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247" +} diff --git a/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json b/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json similarity index 68% rename from backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json rename to backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json index 5889ab4ee7..0f43afed39 100644 --- a/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json +++ b/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", + "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary,\n nt.enabled\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -102,8 +107,9 @@ true, false, false, - true + true, + false ] }, - "hash": "b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48" + "hash": "a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a" } diff --git a/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json b/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json new file mode 100644 index 0000000000..e6fd39901e --- /dev/null +++ b/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin, devops, login_type FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "devops", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "login_type", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901" +} diff --git a/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json b/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json new file mode 100644 index 0000000000..5f2bb07e24 --- /dev/null +++ b/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT enabled\n FROM native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3" +} diff --git a/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json new file mode 100644 index 0000000000..8c6dd010af --- /dev/null +++ b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH queue_metrics as (\n SELECT id, created_at,\n CASE WHEN jsonb_typeof(value) = 'object'\n THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric)\n ELSE value\n END AS value\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "values!", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16" +} diff --git a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json b/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json deleted file mode 100644 index 4e3b39403f..0000000000 --- a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e" -} diff --git a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json deleted file mode 100644 index 4fa871c594..0000000000 --- a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e" -} diff --git a/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json b/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json new file mode 100644 index 0000000000..fa43d27ed8 --- /dev/null +++ b/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH survivors AS (\n SELECT value::text AS rendered FROM resource\n WHERE workspace_id = $1 AND NOT (path = ANY($2::text[]))\n )\n SELECT v.path FROM unnest($3::text[]) AS v(path)\n WHERE EXISTS (\n SELECT 1 FROM survivors s\n WHERE strpos(s.rendered, '\"$var:' || v.path || '\"') > 0\n OR strpos(s.rendered, '\"$jsonvar:' || v.path || '\"') > 0\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d" +} diff --git a/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json b/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json new file mode 100644 index 0000000000..26301c39b2 --- /dev/null +++ b/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM git_sync_ci_test_check c\n WHERE c.concluded AND c.github_posted\n AND c.concluded_at < now() - make_interval(secs => $1)\n AND NOT EXISTS (\n SELECT 1 FROM git_sync_synced_head h\n WHERE h.workspace_id = c.workspace_id\n AND h.repo_resource_path = c.repo_resource_path\n AND h.sha = c.head_sha\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [] + }, + "hash": "a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1" +} diff --git a/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json new file mode 100644 index 0000000000..bc12fdba1f --- /dev/null +++ b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, operator)\n SELECT $1::varchar, u.username, u.email, true, false FROM usr u\n WHERE u.workspace_id = $2 AND u.email = $3 AND u.is_admin AND NOT u.operator AND NOT u.disabled\n AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled)\n AND NOT EXISTS (SELECT 1 FROM usr f WHERE f.workspace_id = $1::varchar AND f.email = $3)\n ON CONFLICT DO NOTHING\n RETURNING username", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8" +} diff --git a/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json new file mode 100644 index 0000000000..313dec3743 --- /dev/null +++ b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock',\n 'model', 'stock', 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily',\n 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c" +} diff --git a/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json b/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json new file mode 100644 index 0000000000..258ee9eaa7 --- /dev/null +++ b/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT repo_resource_path, head_sha FROM git_sync_ci_test_check\n WHERE workspace_id = $1 AND NOT concluded", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "repo_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "head_sha", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352" +} diff --git a/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json b/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json new file mode 100644 index 0000000000..ef7411614b --- /dev/null +++ b/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT consumed_at IS NOT NULL AS \"used!\" FROM login_link WHERE token_hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "used!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c" +} diff --git a/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json b/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json new file mode 100644 index 0000000000..5a9e5da58e --- /dev/null +++ b/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json @@ -0,0 +1,86 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n workspace_id,\n path,\n route_path,\n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n allowed_origins,\n script_path,\n summary,\n description,\n is_flow,\n mode,\n request_type,\n authentication_method,\n http_method,\n static_asset_config,\n edited_by,\n permissioned_as,\n edited_at,\n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, now(), $21, $22, $23, $24\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598" +} diff --git a/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json new file mode 100644 index 0000000000..2e0e072867 --- /dev/null +++ b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf" +} diff --git a/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json new file mode 100644 index 0000000000..41481e2ec7 --- /dev/null +++ b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f" +} diff --git a/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json b/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json new file mode 100644 index 0000000000..1bf06d5366 --- /dev/null +++ b/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO cloud_trial_offer (email, created_by) VALUES ($1, $2)\n ON CONFLICT (email) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961" +} diff --git a/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json new file mode 100644 index 0000000000..222bfc0f82 --- /dev/null +++ b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_unlock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_unlock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385" +} diff --git a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json deleted file mode 100644 index 8d09036772..0000000000 --- a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "suspend", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - false - ] - }, - "hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a" -} diff --git a/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json new file mode 100644 index 0000000000..f67a0f39f2 --- /dev/null +++ b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2) AS \"claimed!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "claimed!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef" +} diff --git a/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json b/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json new file mode 100644 index 0000000000..2991d30d1c --- /dev/null +++ b/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET github_posted = github_posted\n AND check_run_id IS NOT DISTINCT FROM GREATEST(check_run_id, $3),\n check_run_id = GREATEST(check_run_id, $3)\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab" +} diff --git a/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json new file mode 100644 index 0000000000..6b2ec01b23 --- /dev/null +++ b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946" +} diff --git a/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json new file mode 100644 index 0000000000..7476ae9871 --- /dev/null +++ b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2\n AND NOT EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64" +} diff --git a/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json new file mode 100644 index 0000000000..934742008f --- /dev/null +++ b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders',\n 'u/a/wh/analytics/raw_orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24" +} diff --git a/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json b/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json new file mode 100644 index 0000000000..31c628c1a8 --- /dev/null +++ b/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM git_sync_ci_test_check\n WHERE poster_workspace_id = $1 AND repo_resource_path = $2 AND head_sha = $3\n ) as \"exists!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff" +} diff --git a/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json new file mode 100644 index 0000000000..980ba7a36c --- /dev/null +++ b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(git_sync, '{repositories}',\n COALESCE((SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' IN ($2, $3)\n THEN CASE WHEN $4::jsonb = 'null'::jsonb\n THEN elem - 'credential'\n ELSE jsonb_set(elem, '{credential}', $4) END\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem), '[]'::jsonb)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n AND EXISTS (\n SELECT 1 FROM jsonb_array_elements(git_sync->'repositories') AS e\n WHERE e->>'git_repo_resource_path' IN ($2, $3)\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c" +} diff --git a/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json b/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json new file mode 100644 index 0000000000..d64cf09687 --- /dev/null +++ b/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET policy = jsonb_set(\n jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)),\n ARRAY['on_behalf_of_email'], to_jsonb($4::text)\n ) WHERE policy->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905" +} diff --git a/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json b/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json new file mode 100644 index 0000000000..ed6a4f9b5e --- /dev/null +++ b/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO http_trigger (\n path, route_path, route_path_key, script_path, is_flow, workspace_id,\n edited_by, edited_at, extra_perms, authentication_method, http_method,\n static_asset_config, is_static_website, workspaced_route, wrap_body,\n raw_string, allowed_origins, authentication_resource_path, summary, description,\n error_handler_path, error_handler_args, retry, request_type, mode,\n permissioned_as, labels\n )\n SELECT\n path, route_path, route_path_key, script_path, is_flow, $1,\n edited_by, edited_at, extra_perms, authentication_method, http_method,\n static_asset_config, is_static_website, workspaced_route, wrap_body,\n raw_string, allowed_origins, authentication_resource_path, summary, description,\n error_handler_path, error_handler_args, retry, request_type, 'disabled'::TRIGGER_MODE,\n permissioned_as, labels\n FROM http_trigger\n WHERE workspace_id = $2\n AND (workspaced_route IS TRUE OR $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1" +} diff --git a/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json new file mode 100644 index 0000000000..9724a99b18 --- /dev/null +++ b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET ai_sessions_backup_generation = ai_sessions_backup_generation + 1 WHERE workspace_id = $1 AND large_file_storage IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39" +} diff --git a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json deleted file mode 100644 index efd03ae26e..0000000000 --- a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_ids: serde_json::Value", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340" -} diff --git a/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json b/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json new file mode 100644 index 0000000000..c904dc72fd --- /dev/null +++ b/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as \"count!\"\n FROM git_sync_synced_head h\n JOIN v2_job_completed pc ON pc.id = h.job_id\n JOIN v2_job j ON j.workspace_id = h.workspace_id\n AND j.kind IN ('dependencies', 'flowdependencies', 'appdependencies')\n AND j.created_at >= pc.started_at\n AND j.created_at <= COALESCE(h.tests_dispatched_at, now())\n JOIN v2_job_completed c ON c.id = j.id AND c.status IN ('failure', 'canceled')\n WHERE h.workspace_id = $1 AND h.repo_resource_path = $4\n AND h.branch = $3 AND h.sha = $2\n AND h.source = 'pull'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d" +} diff --git a/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json b/backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json similarity index 84% rename from backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json rename to backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json index 0af901b2b8..6a9de7e35b 100644 --- a/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json +++ b/backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary,\n enabled\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -99,8 +104,9 @@ true, false, false, - true + true, + false ] }, - "hash": "15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2" + "hash": "c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1" } diff --git a/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json new file mode 100644 index 0000000000..fee6801a6a --- /dev/null +++ b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n 'u/a/wh/analytics/draft', 'select 3', '{}'),\n ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src',\n 'u/a/wh/analytics/draft_src', 'select 4', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f" +} diff --git a/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json b/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json new file mode 100644 index 0000000000..7bc28f0f39 --- /dev/null +++ b/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json @@ -0,0 +1,185 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n path,\n script_path,\n is_flow,\n route_path,\n authentication_resource_path,\n workspace_id,\n request_type AS \"request_type: _\",\n authentication_method AS \"authentication_method: _\",\n edited_by,\n permissioned_as,\n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n allowed_origins,\n workspaced_route,\n is_static_website,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\",\n mode as \"mode: _\"\n FROM\n http_trigger\n WHERE\n http_method = $1 AND\n (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "authentication_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "request_type: _", + "type_info": { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "authentication_method: _", + "type_info": { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + }, + { + "ordinal": 8, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "static_asset_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "wrap_body", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "raw_string", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "allowed_origins", + "type_info": "TextArray" + }, + { + "ordinal": 14, + "name": "workspaced_route", + "type_info": "Bool" + }, + { + "ordinal": 15, + "name": "is_static_website", + "type_info": "Bool" + }, + { + "ordinal": 16, + "name": "error_handler_path", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "error_handler_args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "retry: _", + "type_info": "Jsonb" + }, + { + "ordinal": 19, + "name": "mode: _", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04" +} diff --git a/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json new file mode 100644 index 0000000000..58300795af --- /dev/null +++ b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem\n FROM workspace_settings, jsonb_array_elements(git_credentials) AS elem\n WHERE workspace_id = $1 AND elem->>'repo_identity' = $2\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15" +} diff --git a/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json new file mode 100644 index 0000000000..2ef26b366b --- /dev/null +++ b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 100000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba" +} diff --git a/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json b/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json new file mode 100644 index 0000000000..cb87d3b3f6 --- /dev/null +++ b/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args->>'repo_url_resource_path' FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201" +} diff --git a/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json new file mode 100644 index 0000000000..cbcb7f18f7 --- /dev/null +++ b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_lock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749" +} diff --git a/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json new file mode 100644 index 0000000000..1e56d65715 --- /dev/null +++ b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact\n WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3\n AND shared_at > now() - ($4::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57" +} diff --git a/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json new file mode 100644 index 0000000000..9c75f7ac97 --- /dev/null +++ b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c" +} diff --git a/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json new file mode 100644 index 0000000000..3f73c2171e --- /dev/null +++ b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO metrics (id, value)\n SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value)\n FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "JsonbArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3" +} diff --git a/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json b/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json new file mode 100644 index 0000000000..30cc0e7092 --- /dev/null +++ b/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)))\n WHERE typ IN ('app', 'raw_app')\n AND value->'policy'->>'on_behalf_of' = $2\n AND value->'policy'->>'on_behalf_of_email' = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e" +} diff --git a/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json b/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json new file mode 100644 index 0000000000..70fd38193c --- /dev/null +++ b/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path FROM script WHERE workspace_id = $1 AND deleted = false AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a" +} diff --git a/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json b/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json new file mode 100644 index 0000000000..b0c2d95175 --- /dev/null +++ b/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5" +} diff --git a/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json new file mode 100644 index 0000000000..748ca6d9ec --- /dev/null +++ b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac" +} diff --git a/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json b/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json new file mode 100644 index 0000000000..669e78201d --- /dev/null +++ b/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_admin, operator, email FROM usr where username = $1 AND workspace_id = $2 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc" +} diff --git a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json similarity index 64% rename from backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json rename to backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json index 400c8d9ee6..340b09ca27 100644 --- a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json +++ b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", "describe": { "columns": [], "parameters": { @@ -11,10 +11,11 @@ "Text", "Varchar", "Varchar", - "Bool" + "Bool", + "Varchar" ] }, "nullable": [] }, - "hash": "5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00" + "hash": "dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606" } diff --git a/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json new file mode 100644 index 0000000000..9622e63a56 --- /dev/null +++ b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id, ws.git_sync\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id\n WHERE NOT w.deleted\n AND ws.git_sync IS NOT NULL\n AND jsonb_typeof(ws.git_sync->'repositories') = 'array'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b" +} \ No newline at end of file diff --git a/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json new file mode 100644 index 0000000000..0b17793580 --- /dev/null +++ b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage IS NOT NULL AS \"has_storage!\" FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_storage!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a" +} diff --git a/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json b/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json new file mode 100644 index 0000000000..9129b657d3 --- /dev/null +++ b/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as \"count!\" FROM v2_job WHERE id = ANY($1::uuid[])", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8" +} diff --git a/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json new file mode 100644 index 0000000000..46a9c8e505 --- /dev/null +++ b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570" +} diff --git a/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json new file mode 100644 index 0000000000..3068e4fe35 --- /dev/null +++ b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool)\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51" +} diff --git a/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json new file mode 100644 index 0000000000..797fee956b --- /dev/null +++ b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77" +} diff --git a/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json b/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json new file mode 100644 index 0000000000..cf4d577dbb --- /dev/null +++ b/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, poster_workspace_id, head_sha, repo_url, repo_resource_path,\n check_run_id\n FROM git_sync_ci_test_check\n WHERE NOT concluded OR NOT github_posted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "head_sha", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "repo_url", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "repo_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "check_run_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + true + ] + }, + "hash": "e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619" +} diff --git a/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json b/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json new file mode 100644 index 0000000000..f8c42256bb --- /dev/null +++ b/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM git_sync_synced_head h\n WHERE h.synced_at < now() - make_interval(secs => $1)\n AND EXISTS (\n SELECT 1 FROM git_sync_synced_head n\n WHERE n.workspace_id = h.workspace_id\n AND n.repo_resource_path = h.repo_resource_path\n AND n.branch = h.branch\n AND n.synced_at > h.synced_at\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [] + }, + "hash": "e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c" +} diff --git a/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json b/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json new file mode 100644 index 0000000000..5d0e560ced --- /dev/null +++ b/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(jc.status::text, 'running') as \"status!\"\n FROM unnest($1::uuid[]) AS run(id)\n LEFT JOIN v2_job_completed jc ON jc.id = run.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357" +} diff --git a/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json new file mode 100644 index 0000000000..a344e64392 --- /dev/null +++ b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ai_sessions_backup_generation", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11" +} diff --git a/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json new file mode 100644 index 0000000000..854ed61857 --- /dev/null +++ b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT n.script_path AS \"script_path!\", n.script_hash, n.job_id AS \"job_id!\"\n FROM dbt_node n\n WHERE n.workspace_id = $1 AND n.asset_path = ANY($2)\n -- The run's snapshot, or the deployed graph when that job stored\n -- none -- a build pins only if it wrote one.\n AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $5)\n THEN $5::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END\n -- The gate, re-decided for every project the walk reaches. That\n -- is what resolving owners in a loop is for: being entitled to\n -- one project is not being entitled to the one that declares a\n -- relation it hands over.\n AND ( $6\n OR n.script_path = ANY($7)\n OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx\n WHERE n.script_path = pfx\n OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) )\n AND CASE\n -- Pinned: which version comes from a job this caller was\n -- already granted, so `script` does not decide THAT -- but\n -- it still decides whether the project may be read, the\n -- same second gate `script_visible` is on the graph. Being\n -- entitled to a run is not being entitled to the SQL\n -- behind it, and column lineage is that SQL's shape. A\n -- version-less row is exempt because it is an editor\n -- buffer, which has no `script` row to ask and reaches\n -- this only through the parse job that wrote it.\n --\n -- One project answers, so a pinned trace never crosses\n -- into another: neither does the graph it annotates.\n WHEN $4::text IS NOT NULL\n THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint\n AND ($3::bigint IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3))\n -- A named version: the deployed one an editor is drawing.\n -- `script` is read under RLS, so this is the visibility\n -- check as well as the existence one. A hash names one\n -- script row, so this arm answers for one project too —\n -- and deliberately: a pin says which stored graph is on\n -- screen, and another project's live graph is not it.\n WHEN $3::bigint IS NOT NULL\n THEN n.script_hash = $3 AND EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3)\n -- Otherwise the version deployed now: an older one's rows\n -- outlive it in `dbt_node` until the sweep, and describe a\n -- project that is no longer what runs. `language` narrows\n -- it the way the graph's own resolution does, so a path\n -- that has since become a script of another kind draws and\n -- explains the same version rather than disagreeing. Read\n -- under RLS, so a project the caller cannot see resolves\n -- to NULL and matches nothing.\n ELSE n.script_hash = (\n SELECT sc.hash FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.language = 'dbt'\n AND sc.deleted = false AND sc.archived = false\n ORDER BY sc.created_at DESC LIMIT 1)\n END", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "job_id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8", + "Text", + "Uuid", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2" +} diff --git a/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json b/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json new file mode 100644 index 0000000000..7020c21b0d --- /dev/null +++ b/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO git_sync_synced_head\n (workspace_id, repo_resource_path, branch, sha, source, job_id)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, repo_resource_path, branch, sha)\n DO UPDATE SET source = EXCLUDED.source, job_id = EXCLUDED.job_id, synced_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0" +} diff --git a/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json new file mode 100644 index 0000000000..1a71c74e9e --- /dev/null +++ b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25" +} diff --git a/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json new file mode 100644 index 0000000000..75526d2db8 --- /dev/null +++ b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b" +} diff --git a/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json new file mode 100644 index 0000000000..1c79a53ab0 --- /dev/null +++ b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14" +} diff --git a/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json b/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json new file mode 100644 index 0000000000..dda976de02 --- /dev/null +++ b/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT repo_url, check_run_id, poster_workspace_id, conclusion,\n created_at, concluded, github_posted, head_ref\n FROM git_sync_ci_test_check\n WHERE workspace_id = $1 AND repo_resource_path = $3 AND head_sha = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "repo_url", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "check_run_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "conclusion", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "concluded", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "github_posted", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "head_ref", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb" +} diff --git a/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json new file mode 100644 index 0000000000..ccb9863853 --- /dev/null +++ b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id',\n 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e" +} diff --git a/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json new file mode 100644 index 0000000000..540d2e0e08 --- /dev/null +++ b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2" +} diff --git a/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json b/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json new file mode 100644 index 0000000000..bf8fa238a0 --- /dev/null +++ b/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE native_trigger\n SET enabled = $1\n WHERE\n workspace_id = $2\n AND service_name = $3\n AND external_id = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76" +} diff --git a/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json new file mode 100644 index 0000000000..6a4b3fe6de --- /dev/null +++ b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71" +} diff --git a/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json new file mode 100644 index 0000000000..c1eb5fa826 --- /dev/null +++ b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e4dc875e49..8ce8430ab3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1724,7 +1724,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1744,7 +1744,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1787,9 +1787,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -1958,18 +1958,18 @@ dependencies = [ [[package]] name = "bon" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" +checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" dependencies = [ "bon-macros", ] [[package]] name = "bon-macros" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" +checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" dependencies = [ "darling 0.24.1", "ident_case", @@ -2095,9 +2095,9 @@ dependencies = [ [[package]] name = "byte-unit" -version = "5.2.5" +version = "5.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a813de7f2bbedb7dce265b64f1cf5908ebe4d56281ece8d847e98113788b9b0" +checksum = "c719d56f7e96194cfc53460976d3ba51c85719747c9c62ed99981847b551152b" dependencies = [ "rust_decimal", "schemars 1.2.2", @@ -2138,9 +2138,9 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" dependencies = [ "proc-macro2", "quote", @@ -2311,9 +2311,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.5" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" dependencies = [ "find-msvc-tools", "jobserver", @@ -2425,9 +2425,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -2435,9 +2435,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -2447,9 +2447,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", @@ -2459,9 +2459,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cmake" @@ -2733,9 +2733,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] @@ -2763,18 +2763,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2782,27 +2782,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm_winapi" @@ -5077,7 +5077,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "rustc_version 0.4.1", ] @@ -5177,9 +5177,9 @@ dependencies = [ [[package]] name = "frostem" -version = "1.20260821.5" +version = "1.20260821.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f" +checksum = "d7d51501290db793146d5005edc29d3feac221da5702a4c63436344841ea88d4" [[package]] name = "fs3" @@ -6194,9 +6194,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] @@ -6463,11 +6463,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -6517,6 +6538,8 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -6524,6 +6547,28 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + [[package]] name = "ident_case" version = "1.0.1" @@ -6640,11 +6685,11 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -6664,9 +6709,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "ipnetwork" @@ -6745,9 +6790,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ "defmt", "jiff-core", @@ -6762,18 +6807,19 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" dependencies = [ "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ "jiff-core", "proc-macro2", @@ -7217,7 +7263,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "lazy_static", "libgssapi-sys", @@ -7262,11 +7308,11 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" +checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "plain", "redox_syscall 0.9.4", @@ -7396,9 +7442,9 @@ dependencies = [ [[package]] name = "lru-slab" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "lscolors" @@ -7906,7 +7952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "btoi", "byteorder", "bytes", @@ -7965,7 +8011,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -7976,7 +8022,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -7988,7 +8034,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8000,7 +8046,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8467,7 +8513,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "once_cell", "onig_sys", @@ -8532,7 +8578,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "foreign-types", "libc", @@ -9077,9 +9123,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" dependencies = [ "memchr", "ucd-trie", @@ -9087,9 +9133,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" dependencies = [ "pest", "pest_generator", @@ -9097,9 +9143,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" dependencies = [ "pest", "pest_meta", @@ -9110,9 +9156,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" dependencies = [ "pest", ] @@ -9485,6 +9531,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -9544,7 +9592,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit 0.25.15+spec-1.1.0", ] [[package]] @@ -9633,7 +9681,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "flate2", "hex", @@ -9647,7 +9695,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", ] @@ -9734,7 +9782,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "getopts", "memchr", "unicase", @@ -9803,9 +9851,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -9823,9 +9871,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "aws-lc-rs", "bytes", @@ -10028,7 +10076,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10162,7 +10210,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10171,7 +10219,7 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10326,11 +10374,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "encoding_rs", "futures-core", @@ -10378,7 +10426,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.5.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "thiserror 2.0.20", "tower-service", @@ -10396,7 +10444,7 @@ dependencies = [ "getrandom 0.2.17", "http 1.5.0", "hyper 1.11.1", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "retry-policies", "thiserror 2.0.20", @@ -10506,7 +10554,7 @@ dependencies = [ "pastey", "pin-project-lite", "rand 0.10.2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp-macros", "schemars 1.2.2", "serde", @@ -10524,9 +10572,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf1c49bd4d52014b94db0877410db273c2008f01628b0252a2e9460ad9b7fda" +checksum = "873b730df6f0a9b74b13eb514e0dca4c2db0d8b68b74af98a2e9bf3f9d436585" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -10741,7 +10789,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10754,7 +10802,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", @@ -11241,7 +11289,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -11254,7 +11302,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -11484,11 +11532,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58", "chrono", "hex", @@ -11505,14 +11553,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -11754,9 +11802,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" dependencies = [ "serde", ] @@ -12017,7 +12065,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "bytes", "chrono", @@ -12061,7 +12109,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "chrono", "crc", @@ -12335,7 +12383,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "is-macro", "num-bigint", "once_cell", @@ -12391,7 +12439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "either", "num-bigint", "phf 0.11.3", @@ -12724,7 +12772,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "enum-as-inner", "libc", @@ -12752,7 +12800,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -13011,11 +13059,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.2" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" dependencies = [ - "unicode-linebreak", + "icu_segmenter", "unicode-width 0.2.2", ] @@ -13197,23 +13245,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] [[package]] name = "tinyvec" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tinyvector" @@ -13559,9 +13599,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.15+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", @@ -13622,6 +13662,7 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", + "flate2", "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", @@ -13641,6 +13682,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "zstd", ] [[package]] @@ -13706,7 +13748,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -14130,12 +14172,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-normalization" version = "0.1.25" @@ -14335,9 +14371,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14352,7 +14388,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "fslock", "gzip-header", "home", @@ -14747,7 +14783,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-nats", @@ -14772,7 +14808,7 @@ dependencies = [ "prometheus", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rumqttc", "rustls 0.23.35", "serde", @@ -14835,7 +14871,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.804.0" +version = "1.812.0" dependencies = [ "async-stream", "async-trait", @@ -14851,7 +14887,7 @@ dependencies = [ "http 1.5.0", "lazy_static", "mime_guess", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -14868,7 +14904,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14881,7 +14917,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "argon2", @@ -14931,7 +14967,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rsa", "rust-embed", "rustls 0.23.35", @@ -15021,7 +15057,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15044,7 +15080,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15061,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15071,7 +15107,7 @@ dependencies = [ "jsonwebtoken 8.3.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15087,7 +15123,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.804.0" +version = "1.812.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15097,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15150,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15136,7 +15172,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15145,7 +15181,7 @@ dependencies = [ "candle-transformers", "hf-hub", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15159,7 +15195,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15175,7 +15211,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15197,7 +15233,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15208,6 +15244,7 @@ dependencies = [ "serde_json", "sql-builder", "sqlx", + "tracing", "uuid", "windmill-api-auth", "windmill-api-workspaces", @@ -15218,7 +15255,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15232,7 +15269,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-nats", @@ -15246,7 +15283,7 @@ dependencies = [ "hmac", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "rumqttc", "serde", @@ -15267,7 +15304,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15292,7 +15329,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15301,7 +15338,7 @@ dependencies = [ "hex", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "semver 1.0.28", "serde", "serde_json", @@ -15320,7 +15357,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15379,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15362,7 +15399,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15373,7 +15410,7 @@ dependencies = [ "lazy_static", "prometheus", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -15400,7 +15437,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15465,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.804.0" +version = "1.812.0" dependencies = [ "lazy_static", "serde", @@ -15440,7 +15477,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.804.0" +version = "1.812.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15464,7 +15501,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15478,7 +15515,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.804.0" +version = "1.812.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15513,7 +15550,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.804.0" +version = "1.812.0" dependencies = [ "chrono", "lazy_static", @@ -15527,7 +15564,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15546,7 +15583,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.812.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15563,7 +15600,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "chrono", "chrono-tz", @@ -15608,7 +15645,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "reqwest-retry", "rsa", @@ -15652,7 +15689,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.804.0" +version = "1.812.0" dependencies = [ "chrono", "futures", @@ -15672,7 +15709,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.804.0" +version = "1.812.0" dependencies = [ "regex", "serde", @@ -15681,13 +15718,15 @@ dependencies = [ "tokio", "tracing", "uuid", + "windmill-audit", "windmill-common", + "windmill-dep-map", "windmill-queue", ] [[package]] name = "windmill-indexer" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15714,7 +15753,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "futures", @@ -15731,7 +15770,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.812.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15747,7 +15786,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -15755,7 +15794,7 @@ dependencies = [ "futures", "http 1.5.0", "oauth2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "serde", "serde_json", @@ -15768,7 +15807,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -15780,7 +15819,7 @@ dependencies = [ "http 1.5.0", "itertools 0.14.0", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -15799,7 +15838,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "arc-swap", @@ -15824,7 +15863,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-stream", @@ -15842,9 +15881,10 @@ dependencies = [ "lazy_static", "object_store", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", + "serial_test", "sqlx", "tempfile", "tokio", @@ -15858,7 +15898,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "futures", @@ -15876,7 +15916,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.812.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15885,7 +15925,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -15897,7 +15937,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde_json", @@ -15909,7 +15949,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "gosyn", @@ -15921,7 +15961,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -15933,7 +15973,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde_json", @@ -15945,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "nu-parser", @@ -15956,7 +15996,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15967,7 +16007,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15979,7 +16019,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15990,7 +16030,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-recursion", @@ -16012,7 +16052,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde_json", @@ -16024,7 +16064,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -16038,7 +16078,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16055,7 +16095,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -16068,7 +16108,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde", @@ -16080,7 +16120,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -16098,7 +16138,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16114,7 +16154,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16130,7 +16170,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -16144,7 +16184,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-recursion", @@ -16165,7 +16205,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "serde_urlencoded", @@ -16183,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "const_format", @@ -16207,7 +16247,7 @@ dependencies = [ "lazy_static", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rustls 0.23.35", "serde", "serde_json", @@ -16223,7 +16263,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.804.0" +version = "1.812.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16234,7 +16274,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-recursion", @@ -16249,7 +16289,7 @@ dependencies = [ "lazy_static", "magic-crypt", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16269,7 +16309,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16333,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16304,7 +16344,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -16326,7 +16366,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16353,7 +16393,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16368,7 +16408,7 @@ dependencies = [ "lazy_static", "quick_cache", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16386,7 +16426,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16446,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16421,7 +16461,7 @@ dependencies = [ "jsonwebtoken 8.3.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -16440,7 +16480,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16476,7 +16516,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16499,7 +16539,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16523,7 +16563,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-nats", @@ -16547,7 +16587,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16582,7 +16622,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16610,7 +16650,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-trait", @@ -16635,10 +16675,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", "itertools 0.14.0", @@ -16654,7 +16694,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-once-cell", @@ -16699,6 +16739,7 @@ dependencies = [ "opentelemetry 0.30.0", "opentelemetry-proto 0.30.0", "oracle", + "parquet", "pem 3.0.6", "pep440_rs", "postgres-native-tls 0.5.3", @@ -16709,7 +16750,7 @@ dependencies = [ "rand 0.9.0", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "rsa", "rust_decimal", @@ -16771,7 +16812,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.804.0" +version = "1.812.0" dependencies = [ "bytes", "futures", @@ -17022,7 +17063,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "widestring", "windows-sys 0.52.0", ] @@ -17471,18 +17512,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", @@ -17539,6 +17580,7 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -17547,6 +17589,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7abb3b92ee..61cc8bd1c1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.804.0" +version = "1.812.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.804.0" +version = "1.812.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -108,6 +108,10 @@ debug = "line-tables-only" [profile.dev.package."*"] debug = false +# The expression parser's fallback call chain exhausts worker stacks without optimization. +[profile.dev.package.php-parser-rs] +opt-level = 1 + [profile.release] lto = "thin" debug = "line-tables-only" @@ -665,6 +669,12 @@ process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" datafusion = "47.0.0" +# The row API only: a dbt engine's parquet index is six string columns, so this +# needs no arrow and no writer. `parquet` is already in the tree with `arrow` for +# every shipped edition (`oss_core`), and cargo unifies the features there; this +# set is what a build WITHOUT object storage compiles. ZSTD is what the engine +# writes today, snap what parquet writers most often default to. +parquet = { version = "55.2.0", default-features = false, features = ["snap", "zstd"] } object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" @@ -691,7 +701,7 @@ async-stream = "^0" opentelemetry = "0.30.0" tracing-opentelemetry = "0.31.0" opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] } -opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto"] } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto", "gzip-tonic", "zstd-tonic"] } opentelemetry-appender-tracing = "0.30.0" opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] } opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 0ebbb1b5ea..317ca12c5e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d33ea730c550cdbc7d050aeb6d40dcef3d134e07 +d252afcc80e77fcc4f9a2a346b80908c8605a6c0 diff --git a/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql b/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql new file mode 100644 index 0000000000..b21e2bb474 --- /dev/null +++ b/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS git_sync_ci_test_check; diff --git a/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql b/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql new file mode 100644 index 0000000000..59fabb890c --- /dev/null +++ b/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql @@ -0,0 +1,39 @@ +-- One "Windmill CI tests" GitHub check run per (fork workspace, repository, PR head commit): +-- the pull_request webhook opens the check in_progress and it is concluded once +-- the fork's CI tests settle, so the results can gate a GitHub PR. +CREATE TABLE git_sync_ci_test_check ( + -- The fork workspace whose CI tests gate the PR: keys the row, and its `ci_test` + -- jobs are what the check reflects. + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + head_sha VARCHAR(64) NOT NULL, + -- The PR's head branch: the check waits until the fork's synced state for this + -- branch (written by its pushes and pulls alike) names `head_sha`. + head_ref VARCHAR(255) NOT NULL, + -- The workspace whose git host credential posts the check: the one that received + -- the pull request webhook (the parent owning the repo hook). + poster_workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + repo_url TEXT NOT NULL, + -- The fork's copy of the repository resource: keys the synced-head lookup, since a + -- fork syncing two repositories names its branch identically in both. + repo_resource_path VARCHAR(255) NOT NULL, + -- NULL when the GitHub check-run creation failed; the poller retries the create. + check_run_id BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + concluded BOOLEAN NOT NULL DEFAULT false, + conclusion TEXT, + concluded_at TIMESTAMPTZ, + -- Decoupled from `concluded` so a failed check-run PATCH is retried by the + -- poller instead of hanging a required check on GitHub. + github_posted BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (workspace_id, repo_resource_path, head_sha) +); + +-- Rows still needing action (create retry, conclusion, timeout, delivery retry). +-- A row drops out only once it is both concluded and delivered to GitHub, so the +-- per-job conclusion hook and the poller sweeper both scan a small live set. +CREATE INDEX idx_git_sync_ci_test_check_pending + ON git_sync_ci_test_check (workspace_id) + WHERE NOT concluded OR NOT github_posted; + +GRANT ALL ON git_sync_ci_test_check TO windmill_user; +GRANT ALL ON git_sync_ci_test_check TO windmill_admin; diff --git a/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql b/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql new file mode 100644 index 0000000000..bee3e6034c --- /dev/null +++ b/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE http_trigger DROP COLUMN allowed_origins; diff --git a/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql b/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql new file mode 100644 index 0000000000..444fc46535 --- /dev/null +++ b/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE http_trigger ADD COLUMN allowed_origins TEXT[]; diff --git a/backend/migrations/20260827203157_login_link.down.sql b/backend/migrations/20260827203157_login_link.down.sql new file mode 100644 index 0000000000..aa26e1eee8 --- /dev/null +++ b/backend/migrations/20260827203157_login_link.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS login_link; diff --git a/backend/migrations/20260827203157_login_link.up.sql b/backend/migrations/20260827203157_login_link.up.sql new file mode 100644 index 0000000000..0be611f408 --- /dev/null +++ b/backend/migrations/20260827203157_login_link.up.sql @@ -0,0 +1,13 @@ +-- Single-use login links minted by a superadmin for one account. Consumed by an +-- unauthenticated GET that mints a session; the row is never a bearer credential itself. +CREATE TABLE login_link ( + token_hash CHAR(64) PRIMARY KEY, + email VARCHAR(255) NOT NULL REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + rd TEXT, + expiration TIMESTAMPTZ NOT NULL, + consumed_at TIMESTAMPTZ, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX login_link_email_idx ON login_link (email); diff --git a/backend/migrations/20260828190901_cloud_trial_offer.down.sql b/backend/migrations/20260828190901_cloud_trial_offer.down.sql new file mode 100644 index 0000000000..0769b91752 --- /dev/null +++ b/backend/migrations/20260828190901_cloud_trial_offer.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cloud_trial_offer; diff --git a/backend/migrations/20260828190901_cloud_trial_offer.up.sql b/backend/migrations/20260828190901_cloud_trial_offer.up.sql new file mode 100644 index 0000000000..d16ea6857c --- /dev/null +++ b/backend/migrations/20260828190901_cloud_trial_offer.up.sql @@ -0,0 +1,11 @@ +-- A pre-approved self-hosted Enterprise trial offered to an account created through a +-- pre-approved invite. No expiry: the offer lasts until a trial or subscription exists. +-- The cascade follows the account out on deletion and rename. A superadmin users-import +-- replaces every account by deleting and reinserting it, which takes these rows with it: +-- the offers, like the onboarding profiles, are recorded by the portal that minted them. +CREATE TABLE cloud_trial_offer ( + email VARCHAR(255) PRIMARY KEY REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + consumed_at TIMESTAMPTZ, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql new file mode 100644 index 0000000000..ca88ec0f1e --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS git_credentials; diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql new file mode 100644 index 0000000000..7ece575cc1 --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql @@ -0,0 +1,8 @@ +-- Server-owned git-sync credentials, one entry per repository the workspace +-- holds a token for, keyed by that repository rather than by a resource naming +-- it: a resource's URL is writable, and its path is not settled while it is +-- being created. +-- Kept out of `git_sync` because that column is copied into forks and returned +-- by the workspace settings API; this one is copied by neither. +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS git_credentials JSONB NOT NULL DEFAULT '[]'::jsonb; diff --git a/backend/migrations/20260904135713_dbt_environment_state.down.sql b/backend/migrations/20260904135713_dbt_environment_state.down.sql new file mode 100644 index 0000000000..dab9025417 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS dbt_environment_state; diff --git a/backend/migrations/20260904135713_dbt_environment_state.up.sql b/backend/migrations/20260904135713_dbt_environment_state.up.sql new file mode 100644 index 0000000000..a49df5b3d1 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.up.sql @@ -0,0 +1,52 @@ +-- The dbt state one project last built into one environment: the `manifest.json` +-- (and the `run_results.json` beside it) that `dbt --defer --state ` resolves +-- an unbuilt `ref()` through. +-- +-- Separate from `dbt_run_state`, which answers a different question. That one is +-- keyed by the executing principal and holds the LAST run whatever its outcome, +-- so `dbt retry` can resume its failures; this one is keyed by environment and +-- holds the last SUCCESSFUL run, because a relation a later run defers to has to +-- exist. Merging them would make a retry resume a run that is not the last one, +-- or a deferral point at relations a failed run never wrote. +CREATE TABLE IF NOT EXISTS dbt_environment_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + -- The workspace warehouse, the dbt target, and the database and schema that + -- target resolves to. All four, because deferring is resolving a relation + -- NAME: a repointed warehouse or a moved schema makes the stored manifest + -- describe relations that are not where this run would look for them, and the + -- run has no other way to notice. A move therefore reads as an environment + -- with no state yet rather than as state that silently no longer fits. + -- + -- TEXT rather than VARCHAR(255): a project bringing its own `profiles.yml` + -- spells its own schema and database, so the length is the project's. + environment TEXT NOT NULL, + -- The run that published it, so a deferring run can say what it deferred to. + job_id UUID NOT NULL, + -- Exactly one home each. A manifest grows with the project and passes a few + -- hundred KB on a handful of models, so a large one goes to the INSTANCE's + -- object storage and this row keeps the key; a small one stays here, where it + -- costs no round trip and works on an instance that has configured no storage + -- at all. The instance's and not the workspace's, because a member can write + -- the workspace bucket under a key of their choosing, and a manifest is what a + -- later run resolves every unbuilt `ref()` through. `run_results.json` is a + -- tenth of the size and takes the same two homes rather than a rule of its own. + manifest TEXT, + manifest_key TEXT, + run_results TEXT, + run_results_key TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, environment), + CONSTRAINT dbt_environment_state_manifest_one_home + CHECK (num_nonnulls(manifest, manifest_key) = 1), + CONSTRAINT dbt_environment_state_run_results_one_home + CHECK (num_nonnulls(run_results, run_results_key) <= 1) +); + +-- No age sweep, unlike the per-run graph rows next door: this table holds one +-- row per script per environment and replaces it in place, so it does not grow +-- with runs, and its reader is every later run of that script — a project that +-- runs monthly must still find last month's state. It goes with the script +-- instead, alongside `dbt_run_state`. +GRANT ALL ON dbt_environment_state TO windmill_user; +GRANT ALL ON dbt_environment_state TO windmill_admin; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.down.sql b/backend/migrations/20260904143633_dbt_column_lineage.down.sql new file mode 100644 index 0000000000..a053a94291 --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS dbt_column_edge; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS column_schema; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.up.sql b/backend/migrations/20260904143633_dbt_column_lineage.up.sql new file mode 100644 index 0000000000..cb463c249f --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.up.sql @@ -0,0 +1,68 @@ +-- Column-level lineage, from the engine's own static analysis. +-- +-- `manifest.json` carries none, which is why decision 14 recorded the feature as +-- unavailable. The edges exist in a different artifact: an engine that does +-- static analysis writes `target/index/dbt.column_lineage.parquet` under +-- `dbt compile --static-analysis strict --write-index`. That pass is opt-in per +-- project (`column_lineage: true`), because strict analysis rejects SQL the +-- default accepts and must never become a silent requirement of running a build. + +-- One column-to-column edge, keyed exactly like `dbt_edge`: a version's graph +-- dies with its version through the composite foreign key, a run's snapshot is +-- keyed by `job_id` with the zero UUID meaning "the version's own graph", and an +-- editor buffer's parse carries a NULL `script_hash` keyed to its preview job. +CREATE TABLE IF NOT EXISTS dbt_column_edge ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + parent_unique_id TEXT NOT NULL, + parent_column TEXT NOT NULL, + child_unique_id TEXT NOT NULL, + child_column TEXT NOT NULL, + -- dbt's own word for how the value travelled: `copy` (passthrough), `mod` + -- (transformed), `scan` (the column was read to produce the ROW rather than + -- the value -- a join key, a `where` predicate, a `group by`). TEXT rather + -- than an enum because the engine treats the set as open: its own reader maps + -- those three and returns anything else verbatim. + lineage_kind TEXT NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Two partial unique indexes rather than a primary key, for the reason + -- 20260801121717 gives: a versioned graph is keyed by its version, a buffer + -- parse by its job alone. `lineage_kind` is part of the key because it is part + -- of the fact: a column that is both projected and used as a predicate for the + -- same output column has a `copy` edge AND a `scan` one, and the digest counts + -- both. Leaving it out let `ON CONFLICT DO NOTHING` drop the second while the + -- digest still claimed it was stored. + CONSTRAINT dbt_column_edge_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_versioned_key + ON dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_editor_key + ON dbt_column_edge (workspace_id, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NULL; + +-- Same age sweep as the other per-run rows, and the same reason there is no +-- foreign key to `v2_job`. +CREATE INDEX IF NOT EXISTS idx_dbt_column_edge_run_age ON dbt_column_edge (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; + +-- The real column schema of a node, which only static analysis knows: an +-- ordered `[{"name": …, "type": …}]`, from `dbt.node_columns.parquet`. +-- +-- Beside `columns` rather than folded into it. `columns` is the DECLARED +-- metadata `manifest.json` carries -- the names an author wrote in `schema.yml` +-- and the prose against them -- and stays exactly that, so a project that +-- documents two of forty columns keeps saying so. This is the other forty, +-- typed, in the order the model produces them. +ALTER TABLE dbt_node ADD COLUMN IF NOT EXISTS column_schema JSONB; + +GRANT ALL ON dbt_column_edge TO windmill_user; +GRANT ALL ON dbt_column_edge TO windmill_admin; diff --git a/backend/migrations/20260908113909_native_trigger_enabled.down.sql b/backend/migrations/20260908113909_native_trigger_enabled.down.sql new file mode 100644 index 0000000000..75f26b22c4 --- /dev/null +++ b/backend/migrations/20260908113909_native_trigger_enabled.down.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger DROP COLUMN IF EXISTS enabled; diff --git a/backend/migrations/20260908113909_native_trigger_enabled.up.sql b/backend/migrations/20260908113909_native_trigger_enabled.up.sql new file mode 100644 index 0000000000..15139498d7 --- /dev/null +++ b/backend/migrations/20260908113909_native_trigger_enabled.up.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT true; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.down.sql b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql new file mode 100644 index 0000000000..32ae30d2bd --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql @@ -0,0 +1,9 @@ +-- Dropping the column drops the primary key and the identity sequence with it, and +-- only marks the column dropped in the catalog rather than rewriting the table, so +-- this takes the ACCESS EXCLUSIVE lock but not the time. + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_column_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_graph_snapshot DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.up.sql b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql new file mode 100644 index 0000000000..399fe6cc5f --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql @@ -0,0 +1,31 @@ +-- These five had neither a PRIMARY KEY nor an explicit REPLICA IDENTITY, which makes +-- PostgreSQL reject UPDATE and DELETE on them under logical replication. +-- `deployment_metadata` and `metrics` are the other two, one migration each after this. +-- +-- The surrogate cannot be swapped for a natural key: every unique index on all five is +-- PARTIAL, split on `script_hash IS NULL`, and a partial index cannot back a primary +-- key. The partial uniques stay; they are what the ON CONFLICT clauses infer. +-- +-- Each ALTER rewrites its table under ACCESS EXCLUSIVE and holds it unavailable for +-- the rewrite. These five share a transaction because each is bounded by what a +-- workspace holds rather than by how long it has run, so none can grow into the one +-- that locks the rest; a transaction holds all its locks until it commits. +-- +-- An instance that cannot afford that lock at startup can set REPLICA IDENTITY FULL +-- on these tables instead, which unblocks replication by itself, and run these +-- idempotent ALTERs in a maintenance window first. + +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_node + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_column_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_graph_snapshot + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql new file mode 100644 index 0000000000..3ce4ae52a6 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE deployment_metadata DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql new file mode 100644 index 0000000000..951253ce64 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql @@ -0,0 +1,13 @@ +-- The sixth of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the one table in the set with no retention +-- sweep -- rows accumulate per deployed script hash, flow version and app version for +-- the life of the instance -- so on an instance that upgrades after years of deploys +-- its ACCESS EXCLUSIVE rewrite is the one that could hold the others locked. +-- +-- No natural key: each row is a script, flow OR app deployment, and the three unique +-- indexes are partial on exactly that split, so none of them covers every row. + +ALTER TABLE deployment_metadata + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.down.sql b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql new file mode 100644 index 0000000000..1ad428dc02 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE metrics DROP COLUMN IF EXISTS row_id; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.up.sql b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql new file mode 100644 index 0000000000..b863035990 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql @@ -0,0 +1,12 @@ +-- The last of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the largest (~400 MB / 750k rows on the +-- instance this was measured on, a steady state: `queue_%` rows, which are nearly all +-- of them, are swept at 14 days) and its ALTER rewrites it under ACCESS EXCLUSIVE. One +-- migration is one transaction, so alone it holds no lock on the others as it rewrites. +-- +-- The surrogate cannot be called `id` -- `metrics.id` holds the metric NAME. + +ALTER TABLE metrics + ADD COLUMN IF NOT EXISTS row_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql b/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql new file mode 100644 index 0000000000..a089085444 --- /dev/null +++ b/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS git_sync_synced_head; diff --git a/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql b/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql new file mode 100644 index 0000000000..a0c81d2671 --- /dev/null +++ b/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql @@ -0,0 +1,24 @@ +-- One row per commit a workspace has come to reflect on a branch, written when a +-- pull of that commit succeeds or a deploy push produces it. The "Windmill CI +-- tests" PR check reads it to know when a workspace reflects a PR head, and +-- records the head's CI test runs on it. Kept apart from `workspace_settings.git_sync.auto_pull.last_synced_sha`, +-- which decides whether the next poll pulls and is client-round-tripped settings. +CREATE TABLE git_sync_synced_head ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- Repository resource path without its `$res:` prefix. + repo_resource_path VARCHAR(255) NOT NULL, + branch VARCHAR(255) NOT NULL, + sha VARCHAR(64) NOT NULL, + -- 'pull' rows name the pull job; 'push' rows the deploy push job. + source VARCHAR(4) NOT NULL CHECK (source IN ('pull', 'push')), + job_id UUID, + synced_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- The head's own CI test suite, dispatched once the workspace reflects it and its + -- dependency jobs settled: the "Windmill CI tests" check reads exactly these runs. + tests_dispatched_at TIMESTAMPTZ, + ci_test_job_ids UUID[], + PRIMARY KEY (workspace_id, repo_resource_path, branch, sha) +); + +GRANT ALL ON git_sync_synced_head TO windmill_user; +GRANT ALL ON git_sync_synced_head TO windmill_admin; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql new file mode 100644 index 0000000000..3405d5a681 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql @@ -0,0 +1,7 @@ +DROP INDEX IF EXISTS index_app_version_on_app_id; + +DROP INDEX IF EXISTS index_app_script_on_app; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql new file mode 100644 index 0000000000..6426aec594 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql @@ -0,0 +1,32 @@ +-- The FK columns that cascade when a workspace's apps and flows are deleted. Unindexed, +-- Postgres seq-scans the whole child table once per deleted parent row, making a workspace +-- delete cost O(apps and flows deleted x rows in the instance). Fork deletion is where that +-- bites: a fork clones its parent's apps, flows and entire app version history. +-- +-- The workspace_runnable_dependencies pair are partial because the table's check constraint +-- makes app_path and flow_path mutually exclusive, halving each index -- the cascade's +-- equality on the path proves the predicate. The table's existing path indexes are partial on +-- script_hash, which the cascade does not constrain, so they cannot serve it. +-- +-- Dropped before built: a failed concurrent build leaves an invalid index that IF NOT EXISTS +-- would accept forever, unused by the planner yet still maintained on every write. +-- +-- No statement separators outside the statements below, comments included: the CONCURRENTLY +-- rewrite in windmill-api/src/db.rs splits the file on them and would run comment text as SQL. +DROP INDEX IF EXISTS index_app_version_on_app_id; + +CREATE INDEX index_app_version_on_app_id ON app_version (app_id); + +DROP INDEX IF EXISTS index_app_script_on_app; + +CREATE INDEX index_app_script_on_app ON app_script (app); + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_app_path + ON workspace_runnable_dependencies (app_path, workspace_id) WHERE app_path IS NOT NULL; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_flow_path + ON workspace_runnable_dependencies (flow_path, workspace_id) WHERE flow_path IS NOT NULL; diff --git a/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql b/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql new file mode 100644 index 0000000000..3c67031c9f --- /dev/null +++ b/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cloud_onboarding_profile; diff --git a/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql b/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql new file mode 100644 index 0000000000..8e295ea6c6 --- /dev/null +++ b/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql @@ -0,0 +1,9 @@ +-- Context an invite carried about the account's owner, written at provisioning and read by +-- onboarding to tailor itself (skip the source question it knows the answer to, later +-- template picks and starter prompts). Free-form JSON so new fields need no migration. +CREATE TABLE cloud_onboarding_profile ( + email VARCHAR(255) PRIMARY KEY REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + profile JSONB NOT NULL, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql new file mode 100644 index 0000000000..f615c55463 --- /dev/null +++ b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql @@ -0,0 +1,6 @@ +-- Add down migration script here +-- Nothing to undo. The up migration gives a policy that only ever carried the address the +-- principal it runs as, and rewrites an address that disagreed with its principal. The previous +-- version reads both halves, so both results are correct for it too, and the addresses replaced +-- named an account other than the one the app runs as. +SELECT 1; diff --git a/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql new file mode 100644 index 0000000000..fe97d7529c --- /dev/null +++ b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql @@ -0,0 +1,100 @@ +-- Add up migration script here +-- `policy.on_behalf_of` becomes the authority for an app's identity: the address beside it is +-- written through from it on every save, so the two can no longer name different accounts. +-- +-- The address key is deliberately NOT removed here, and is still written: a replica predating +-- the derive-when-absent fallback errors outright when it is missing, which would 400 every +-- anonymous, publisher and guest app served by one that has not yet rolled over. Removing the +-- key is a follow-up, per docs/app-policy-email-removal.md. +-- +-- What is left is the data written before that rule. A policy that only ever had the address has +-- no principal to run as, so give it one. A policy whose halves disagree was stored as a client +-- sent it; reads return that pair and a redeploy that keeps the identity sends it back, where the +-- pair check rejects it. So once every policy has a principal, rewrite its address from it. + +-- Mirrors `users::username_to_permissioned_as`: an email-shaped username is its own principal +-- unless it contains a slash, which a reader would split on, and a legacy `group-*` username is +-- the group it names. +CREATE OR REPLACE FUNCTION pg_temp.username_to_permissioned_as(name VARCHAR) +RETURNS VARCHAR AS $$ + SELECT CASE + WHEN $1 LIKE '%@%' AND $1 LIKE '%/%' THEN 'u/' || $1 + WHEN $1 LIKE '%@%' THEN $1 + WHEN $1 LIKE 'group-%' THEN 'g/' || substr($1, 7) + ELSE 'u/' || $1 + END; +$$ LANGUAGE SQL IMMUTABLE; + +-- Mirrors `users::permissioned_as_from_email`: a real account wins over the synthetic group +-- namespace, which is not reserved and may be a user's own address. `pg_temp` lives for the +-- whole session and migrations share one connection, so an identically-named helper from an +-- earlier migration is still in scope: replace it, and drop this one at the end. +CREATE OR REPLACE FUNCTION pg_temp.permissioned_as_from_email(w_id VARCHAR, email VARCHAR) +RETURNS VARCHAR AS $$ + SELECT COALESCE( + (SELECT pg_temp.username_to_permissioned_as(u.username) + FROM usr u WHERE u.workspace_id = $1 AND u.email = $2), + -- A superadmin acting outside their workspaces has no usr row. + (SELECT pg_temp.username_to_permissioned_as(COALESCE(p.username, p.email)) + FROM password p WHERE p.email = $2 AND p.super_admin), + (SELECT 'g/' || g.name FROM group_ g + WHERE g.workspace_id = $1 + AND $2 = 'group-' || g.name || '@windmill.dev') + ); +$$ LANGUAGE SQL STABLE; + +-- Mirrors `users::get_email_from_permissioned_as`, except that a `u/` principal naming nobody +-- yields NULL rather than the synthetic `@unknown.windmill.dev` address, so that row is left as +-- it is instead of losing the one address it had. +CREATE OR REPLACE FUNCTION pg_temp.email_from_permissioned_as(w_id VARCHAR, principal VARCHAR) +RETURNS VARCHAR AS $$ + SELECT CASE + WHEN $2 LIKE 'u/%' THEN COALESCE( + (SELECT u.email FROM usr u WHERE u.workspace_id = $1 AND u.username = substr($2, 3)), + (SELECT p.email FROM password p + WHERE (p.username = substr($2, 3) OR p.email = substr($2, 3)) AND p.super_admin + ORDER BY p.email LIMIT 1)) + WHEN $2 LIKE 'g/%' THEN 'group-' || substr($2, 3) || '@windmill.dev' + ELSE $2 + END; +$$ LANGUAGE SQL STABLE; + +-- A policy naming only the address predates the principal being written to it. +-- A principal wider than `v2_job.permissioned_as` could not be enqueued, so it is not recorded +-- at all — the app falls back to erroring on anonymous execution until someone picks an identity +-- the deploy path accepts. Same cap and reason as the sibling migration 20260801043001. +UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], + to_jsonb(pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email'))) + WHERE policy->>'on_behalf_of' IS NULL + AND pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email') IS NOT NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email')) <= 55; + +-- App drafts carry a copy of the policy and are deployed from it, so they need the same. +UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], + to_jsonb(pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email')))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' IS NULL + AND pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email') IS NOT NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email')) <= 55; + +-- The address a save now writes, applied to the rows saved before. Execution already takes a `u/` +-- principal's own address, so this only changes what runs for a `g/` or bare principal, whose +-- stored address decided the superadmin flag and instance groups: those now follow the principal. +UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of_email'], + to_jsonb(pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of'))) + WHERE policy->>'on_behalf_of' IS NOT NULL + AND pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of') IS NOT NULL + AND policy->>'on_behalf_of_email' + IS DISTINCT FROM pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of'); + +UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], + to_jsonb(pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of')))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' IS NOT NULL + AND pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of') IS NOT NULL + AND value->'policy'->>'on_behalf_of_email' + IS DISTINCT FROM pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of'); + +DROP FUNCTION pg_temp.permissioned_as_from_email(VARCHAR, VARCHAR); +DROP FUNCTION pg_temp.email_from_permissioned_as(VARCHAR, VARCHAR); +DROP FUNCTION pg_temp.username_to_permissioned_as(VARCHAR); diff --git a/backend/migrations/20260911085230_notify_user_email_change.down.sql b/backend/migrations/20260911085230_notify_user_email_change.down.sql new file mode 100644 index 0000000000..7f6f494115 --- /dev/null +++ b/backend/migrations/20260911085230_notify_user_email_change.down.sql @@ -0,0 +1,8 @@ +-- Add down migration script here +DROP TRIGGER IF EXISTS password_superadmin_delete_trigger ON password; +DROP TRIGGER IF EXISTS password_superadmin_insert_trigger ON password; +DROP TRIGGER IF EXISTS password_superadmin_update_trigger ON password; +DROP TRIGGER IF EXISTS usr_email_update_trigger ON usr; +DROP TRIGGER IF EXISTS usr_email_change_trigger ON usr; +DROP FUNCTION IF EXISTS notify_superadmin_identity_change(); +DROP FUNCTION IF EXISTS notify_usr_email_change(); diff --git a/backend/migrations/20260911085230_notify_user_email_change.up.sql b/backend/migrations/20260911085230_notify_user_email_change.up.sql new file mode 100644 index 0000000000..f4e4a818f9 --- /dev/null +++ b/backend/migrations/20260911085230_notify_user_email_change.up.sql @@ -0,0 +1,85 @@ +-- Add up migration script here +-- Emit a notify_event so every process evicts its cached `permissioned_as` -> address mapping +-- (windmill-common EMAIL_CACHE) at its next notify-event poll, rather than serving the old +-- address for the rest of the TTL. Authorization does not rest on this: +-- `fetch_authed_from_permissioned_as` re-resolves the address from the principal's live binding. +-- SECURITY DEFINER so the INSERT runs as the function owner rather than the invoking +-- windmill_user/windmill_admin role, matching the other notify_* triggers. +CREATE OR REPLACE FUNCTION notify_usr_email_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) + VALUES ( + 'notify_user_email_change', + COALESCE(NEW.workspace_id, OLD.workspace_id) || ':' || COALESCE(NEW.username, OLD.username) + ); + -- A rename leaves the OLD username cached against this account's address; evict both keys. + IF TG_OP = 'UPDATE' AND NEW.username IS DISTINCT FROM OLD.username THEN + INSERT INTO notify_event (channel, payload) + VALUES ('notify_user_email_change', OLD.workspace_id || ':' || OLD.username); + END IF; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +-- INSERT matters too: a lookup that resolved to nobody is cached as the synthetic +-- `{username}@unknown.windmill.dev`, so creating the row has to drop that entry. +CREATE TRIGGER usr_email_change_trigger +AFTER INSERT OR DELETE ON usr +FOR EACH ROW +EXECUTE FUNCTION notify_usr_email_change(); + +CREATE TRIGGER usr_email_update_trigger +AFTER UPDATE OF email, username ON usr +FOR EACH ROW +WHEN (OLD.email IS DISTINCT FROM NEW.email OR OLD.username IS DISTINCT FROM NEW.username) +EXECUTE FUNCTION notify_usr_email_change(); + +-- A superadmin acting outside their workspaces resolves through `password` instead, and that row +-- names no workspace of its own. The `*:` payload says so: the reader drops that name's entry in +-- every workspace rather than the whole cache, which would undo the caching on an instance that +-- rewrites these rows in bulk. Confined to superadmins because they are the only accounts the +-- `usr` triggers above cannot cover. +CREATE OR REPLACE FUNCTION notify_superadmin_identity_change() +RETURNS TRIGGER AS $$ +DECLARE + names TEXT[] := '{}'; +BEGIN + -- Every alias the principal can be spelled as: `resolve_username_to_email` matches a `u/` + -- principal against `username` OR `email`, and whichever string the caller passed is the key + -- it cached under, so one account can hold a live entry under either. Old and new of each, + -- because a change to one leaves the other's entry behind. + IF TG_OP <> 'DELETE' THEN names := names || ARRAY[NEW.username, NEW.email]; END IF; + IF TG_OP <> 'INSERT' THEN names := names || ARRAY[OLD.username, OLD.email]; END IF; + INSERT INTO notify_event (channel, payload) + SELECT DISTINCT 'notify_user_email_change', '*:' || n + FROM unnest(names) AS n + WHERE n IS NOT NULL; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +-- `super_admin` is half of what the fallback matches on, so gaining or losing it moves the +-- mapping as surely as the address does: a demotion leaves the real address cached where the +-- truth is now synthetic, and a promotion leaves that synthetic one cached in place of a real +-- account. `OLD.super_admin OR NEW.super_admin` is what catches both directions. +CREATE TRIGGER password_superadmin_update_trigger +AFTER UPDATE OF email, username, super_admin ON password +FOR EACH ROW +WHEN ((OLD.super_admin OR NEW.super_admin) + AND (OLD.email IS DISTINCT FROM NEW.email + OR OLD.username IS DISTINCT FROM NEW.username + OR OLD.super_admin IS DISTINCT FROM NEW.super_admin)) +EXECUTE FUNCTION notify_superadmin_identity_change(); + +CREATE TRIGGER password_superadmin_insert_trigger +AFTER INSERT ON password +FOR EACH ROW +WHEN (NEW.super_admin) +EXECUTE FUNCTION notify_superadmin_identity_change(); + +CREATE TRIGGER password_superadmin_delete_trigger +AFTER DELETE ON password +FOR EACH ROW +WHEN (OLD.super_admin) +EXECUTE FUNCTION notify_superadmin_identity_change(); diff --git a/backend/migrations/20260914131148_resource_type_display_name.down.sql b/backend/migrations/20260914131148_resource_type_display_name.down.sql new file mode 100644 index 0000000000..57c9ffabc8 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.down.sql @@ -0,0 +1 @@ +ALTER TABLE resource_type DROP COLUMN display_name; diff --git a/backend/migrations/20260914131148_resource_type_display_name.up.sql b/backend/migrations/20260914131148_resource_type_display_name.up.sql new file mode 100644 index 0000000000..213622ef94 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.up.sql @@ -0,0 +1,24 @@ +-- The name a product goes by, beside the identifier a resource references: `gsheets` is +-- "Google Sheets". Null where nobody named the type; readers derive a label from the name. +ALTER TABLE resource_type ADD COLUMN display_name VARCHAR(100); + +-- The names the hub carries today, so existing instances show them before any sync. Only in +-- admins, where hub resource types live and every workspace reads them from. +UPDATE resource_type SET display_name = v.display_name +FROM (VALUES + ('bamboo_hr', 'BambooHR'), + ('cacertificate', 'CA certificate'), + ('deep_infra', 'DeepInfra'), + ('gcal', 'Google Calendar'), + ('gdocs', 'Google Docs'), + ('gdrive', 'Google Drive'), + ('gforms', 'Google Forms'), + ('gsheets', 'Google Sheets'), + ('gworkspace', 'Google Workspace'), + ('sensortower', 'Sensor Tower'), + ('snowflake_oauth', 'Snowflake (OAuth)'), + ('their_stack', 'TheirStack') +) AS v(name, display_name) +WHERE resource_type.workspace_id = 'admins' + AND resource_type.name = v.name + AND resource_type.display_name IS NULL; diff --git a/backend/migrations/20260914135805_ai_shared_artifact.down.sql b/backend/migrations/20260914135805_ai_shared_artifact.down.sql new file mode 100644 index 0000000000..04fa92db6c --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS ai_shared_artifact; diff --git a/backend/migrations/20260914135805_ai_shared_artifact.up.sql b/backend/migrations/20260914135805_ai_shared_artifact.up.sql new file mode 100644 index 0000000000..6daa3e5daf --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.up.sql @@ -0,0 +1,32 @@ +-- A copy of an AI session artifact that its author explicitly shared with the workspace. +-- Artifacts otherwise live only in the author's browser; this row exists only while the +-- share does, and the monitor deletes it once `shared_at` falls outside +-- AI_SHARED_ARTIFACT_RETENTION_SECS. +CREATE TABLE ai_shared_artifact ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- The browser-side artifact id. Unique per author so sharing the same artifact again + -- moves its one link forward rather than minting a second one. + artifact_id VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL, + created_by VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + kind VARCHAR(10) NOT NULL CHECK (kind IN ('md', 'html')), + version INTEGER NOT NULL, + content TEXT NOT NULL, + -- Reset on every re-share: retention counts from the last time the author shared it. + shared_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (workspace_id, email, artifact_id) +); + +CREATE INDEX idx_ai_shared_artifact_shared_at ON ai_shared_artifact (shared_at); + +GRANT ALL ON ai_shared_artifact TO windmill_admin; +GRANT ALL ON ai_shared_artifact TO windmill_user; + +-- The handlers go through the raw pool and scope every query to the workspace themselves. +-- An admin-only policy is the backstop for a future query that reaches this table through +-- UserDB. +ALTER TABLE ai_shared_artifact ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_shared_artifact FOR ALL TO windmill_admin USING (true); diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql new file mode 100644 index 0000000000..2662a9f709 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS ai_sessions_backup_generation; diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql new file mode 100644 index 0000000000..a62caefa60 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql @@ -0,0 +1,4 @@ +-- Bumped by every workspace key rotation: the AI session backups in the workspace storage +-- live under a prefix named by it, so a rotation moves to a fresh prefix and the previous +-- ones can be deleted at leisure without ever touching live objects. +ALTER TABLE workspace_settings ADD COLUMN ai_sessions_backup_generation BIGINT NOT NULL DEFAULT 0; diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql new file mode 100644 index 0000000000..c749fbe1b7 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql @@ -0,0 +1,9 @@ +DELETE FROM workspace_runnable_dependencies WHERE runnable_is_agent; + +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, workspace_id) + WHERE script_hash IS NULL; + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN runnable_is_agent; diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql new file mode 100644 index 0000000000..eab08e5c17 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql @@ -0,0 +1,21 @@ +-- A flow step linked to a saved agent (an `ai_agent` resource) is recorded next to the scripts and +-- subflows the flow runs, so renaming the agent can name the flows it would break. An agent row is +-- neither a script nor a flow: readers of script usages have to exclude it. +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN runnable_is_agent BOOLEAN NOT NULL DEFAULT false; + +-- A script step and a linked agent can share a path. Without the flag in the key, the second +-- insert's ON CONFLICT DO NOTHING would silently drop one of the two rows. +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) + WHERE script_hash IS NULL; + +-- The worker only records a flow when it is next deployed, so seed the ones already linking an +-- agent from their current value. +INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) +SELECT DISTINCT f.path, agent_ref #>> '{}', false, true, f.workspace_id +FROM flow f +CROSS JOIN LATERAL jsonb_path_query(f.value, 'lax $.** ? (@.type == "aiagent" && @.agent.type() == "string").agent') AS agent_ref +ON CONFLICT DO NOTHING; diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 3f8c265a4a..bc2498a943 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -241,6 +241,7 @@ } }, "snowflake_oauth": { + "resource_fields": ["database", "warehouse", "role", "schema"], "connect_config_template": { "display_name": "Snowflake", "label": "Snowflake Account Identifier", diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index b0ed12fb1a..271ad4835b 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -694,6 +694,107 @@ pub fn remove_pinned_imports(code: &str) -> anyhow::Result { Ok(content) } +/// Spans of the string literals naming a loaded module: `import`/`export … from` sources and the +/// argument of a dynamic `import()`. A `require()` call is left out: `require` is an ordinary +/// binding a script can shadow, so its argument is not known to be a module. +struct ImportSpecifierSpans(Vec); + +impl Visit for ImportSpecifierSpans { + noop_visit_type!(); + + fn visit_import_decl(&mut self, n: &swc_ecma_ast::ImportDecl) { + self.0.push(n.src.span); + } + + fn visit_export_all(&mut self, n: &swc_ecma_ast::ExportAll) { + self.0.push(n.src.span); + } + + fn visit_named_export(&mut self, n: &swc_ecma_ast::NamedExport) { + if let Some(src) = &n.src { + self.0.push(src.span); + } + } + + fn visit_call_expr(&mut self, n: &swc_ecma_ast::CallExpr) { + if let (swc_ecma_ast::Callee::Import(_), Some(arg)) = (&n.callee, n.args.first()) { + if let (None, Expr::Lit(Lit::Str(s))) = (arg.spread, &*arg.expr) { + self.0.push(s.span); + } + } + n.visit_children_with(self); + } +} + +/// Drops the `@version` from each pinned module specifier (`pkg@1.2.3/sub` -> `pkg/sub`), +/// rewriting only the specifier literals. Unlike [`remove_pinned_imports`], the same text +/// elsewhere, such as a string the script returns, stays as written. +pub fn remove_pinned_import_specifiers(code: &str) -> anyhow::Result { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file( + FileName::Custom("main.d.ts".into()).into(), + code.to_string(), + ); + let mut tss = TsSyntax::default(); + tss.tsx = true; + tss.no_early_errors = true; + let lexer = Lexer::new( + Syntax::Typescript(tss), + Default::default(), + StringInput::from(&*fm), + None, + ); + let module = Parser::new_from(lexer).parse_module().map_err(|e| { + anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {e:?}") + })?; + let mut specifiers = ImportSpecifierSpans(vec![]); + specifiers.visit_module(&module); + specifiers.0.sort_by_key(|s| s.lo); + + // Spans index the parsed source, which the source map stripped of any UTF-8 BOM. + let bom = if code.starts_with('\u{feff}') { + '\u{feff}'.len_utf8() + } else { + 0 + }; + let offset = + |pos: swc_common::BytePos| pos.0.checked_sub(fm.start_pos.0).map(|o| bom + o as usize); + let mut content = String::with_capacity(code.len()); + let mut copied = 0; + for span in specifiers.0 { + // A span covers the literal's quotes. One that does not land on a matching pair is left + // as written rather than risk rewriting the wrong bytes. + let (Some(open), Some(close)) = ( + offset(span.lo), + offset(span.hi).and_then(|e| e.checked_sub(1)), + ) else { + continue; + }; + let quote = code.as_bytes().get(open); + if open >= close + || open < copied + || !matches!(quote, Some(b'"' | b'\'')) + || code.as_bytes().get(close) != quote + { + continue; + } + let Some(specifier) = code.get(open + 1..close) else { + continue; + }; + let unpinned = IMPORTS_VERSION.captures(specifier).and_then(|x| { + x.get(1) + .map(|y| format!("{}{}", y.as_str(), x.get(2).map_or("", |z| z.as_str()))) + }); + if let Some(unpinned) = unpinned.filter(|u| u != specifier) { + content.push_str(&code[copied..open + 1]); + content.push_str(&unpinned); + copied = close; + } + } + content.push_str(&code[copied..]); + Ok(content) +} + fn resolve_type_ref(type_resolver: &HashMap, typ: &mut Typ) { let mut visited = std::collections::HashSet::new(); resolve_type_ref_with_visited(type_resolver, typ, &mut visited); diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 4309018fb4..4206dd68f8 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -4,6 +4,7 @@ mod tests { use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; use windmill_parser_ts::{ parse_deno_signature, parse_expr_for_imports, parse_relative_imports, + remove_pinned_import_specifiers, }; #[test] @@ -33,6 +34,46 @@ mod tests { ); } + #[test] + fn test_remove_pinned_import_specifiers_rewrites_only_specifiers() { + let code = r#"// héllo +import a from "pkg@1.2.3"; +import b from "@scope/pkg@^2/sub"; +export * from "other@3"; +import rel from "./helper"; +const c = await import("dyn@4"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"#; + assert_eq!( + remove_pinned_import_specifiers(code).unwrap(), + r#"// héllo +import a from "pkg"; +import b from "@scope/pkg/sub"; +export * from "other"; +import rel from "./helper"; +const c = await import("dyn"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"# + ); + assert_eq!( + remove_pinned_import_specifiers("\u{feff}import a from 'pkg@1';").unwrap(), + "\u{feff}import a from 'pkg';" + ); + assert_eq!( + remove_pinned_import_specifiers( + "// a\r\n// b\r\nimport a from \"pkg@1\";\r\nimport b from 'x@2';" + ) + .unwrap(), + "// a\r\n// b\r\nimport a from \"pkg\";\r\nimport b from 'x';" + ); + } + #[test] fn test_parse_empty_main_signature() { let code = r#" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 9ed900214c..f760f0aaf7 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.812.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.812.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.812.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.812.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index cbb7102820..8ea14f7df4 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.804.0" +version = "1.812.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs index 24f1e2c39d..36617c5639 100644 --- a/backend/parsers/windmill-parser-yaml/src/dbt.rs +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -58,6 +58,31 @@ impl DbtEngine { pub fn emits_node_events(&self) -> bool { matches!(self, DbtEngine::DbtCore1x) } + + /// Whether the engine's CLI has `--write-index`, the flag that writes the + /// parquet index column lineage lives in. False for 1.x, whose Python CLI + /// has no such option. + /// + /// True is not a promise that the artifact appears: `dbt-core` 2.0.0-alpha.5 + /// accepts the flag, declares the views over `dbt.column_lineage` in its own + /// `views.sql`, and writes neither that parquet nor `dbt.node_columns`. Only + /// Fusion does today. Attempting the pass on both is what lets a later 2.x + /// release pick the feature up with no change here. + pub fn writes_column_index(&self) -> bool { + !matches!(self, DbtEngine::DbtCore1x) + } + + /// Whether the engine has `--defer-state`, the deferral-only half of + /// `--state`. + /// + /// It matters on one command. `dbt retry` reads the run it resumes from + /// `--state`, so an engine with only that flag cannot be told to defer and + /// to resume from the job's own results at once: handed the deferral's + /// directory, it resumes the all-green run stored there and rebuilds + /// nothing. Only dbt-core 1.x separates the two. + pub fn has_defer_state_flag(&self) -> bool { + matches!(self, DbtEngine::DbtCore1x) + } } /// How the warehouse connection is supplied. Both paths are supported @@ -127,6 +152,18 @@ pub struct DbtDescriptor { pub selector: Option, #[serde(default)] pub test_behavior: DbtTestBehavior, + /// Ingest column-to-column lineage and the real column schemas, from the + /// engine's static analysis. + /// + /// Opt-in, and it has to be: the artifact only appears under + /// `--static-analysis strict`, which rejects SQL the default accepts (an + /// unresolvable identifier is an error there and compiles fine otherwise). + /// Turning it on for everyone would make a stricter dialect the price of + /// deploying a dbt project. It is a separate `dbt compile` pass, so nothing + /// it decides can change what a build does; a project it cannot analyze + /// keeps the graph it has today. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub column_lineage: bool, /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are /// all normal — so values keep their YAML type; only string leaves carry /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing @@ -137,6 +174,16 @@ pub struct DbtDescriptor { pub threads: Option, #[serde(default)] pub full_refresh: bool, + /// Resolve a `ref()` a run does not build through the state the last + /// successful run of this environment published, rather than through the + /// schema that run writes into. + /// + /// Only the default for the `build` block's own `defer`, since the choice is + /// per run: the run that publishes an environment's state and the run that + /// defers to it are two invocations of ONE script (decision 6), so a project + /// that could only defer by descriptor could never populate what it reads. + #[serde(default)] + pub defer: bool, /// Automatic in-job retry of the nodes a build failed on. /// /// dbt already confines a failure to its own subtree, and `dbt retry` @@ -258,6 +305,7 @@ pub const RESERVED_ARG_NAMES: &[&str] = &[ "exclude", "vars", "full_refresh", + "defer", "dbt_command", "dbt_retry_job", "model", @@ -345,15 +393,26 @@ fn command_variants(d: &DbtDescriptor) -> Vec<(&'static str, Vec)> { "build", selection() .into_iter() - .chain([Arg { - name: "full_refresh".to_string(), - otyp: None, - typ: Typ::Bool, - has_default: true, - default: Some(serde_json::json!(d.full_refresh)), - oidx: None, - otyp_inferred: false, - }]) + .chain([ + Arg { + name: "full_refresh".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.full_refresh)), + oidx: None, + otyp_inferred: false, + }, + Arg { + name: "defer".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.defer)), + oidx: None, + otyp_inferred: false, + }, + ]) .collect(), ), ( @@ -546,7 +605,9 @@ fn property_of(arg: &Arg) -> serde_json::Value { ), "select" => Some( "dbt selection syntax, e.g. `tag:nightly`, `stg_orders+`, \ - `config.materialized:incremental`. Empty runs the descriptor's own selection.", + `config.materialized:incremental`. `state:modified+` and `result:error+` \ + compare against the state a previous run published, so they need `defer` on. \ + Empty runs the descriptor's own selection.", ), "exclude" => Some("Nodes to leave out of the selection above, same syntax."), "vars" => Some( @@ -554,6 +615,11 @@ fn property_of(arg: &Arg) -> serde_json::Value { exist makes this run store its own graph rather than the deployed one.", ), "full_refresh" => Some("Rebuild incremental models from scratch instead of appending."), + "defer" => Some( + "Resolve a `ref()` this run does not build to the relation the last successful \ + run of this warehouse and target published, instead of to the schema this run \ + writes into.", + ), "model" => Some( "The model to preview, by name — `stg_orders`, or `my_package.stg_orders` when \ two packages share a name. Any dbt selector resolving to ONE node works.", @@ -687,8 +753,15 @@ full_refresh: true }; let (build, build_args) = of("build"); - assert_eq!(build_args, ["exclude", "full_refresh", "select", "vars"]); + assert_eq!( + build_args, + ["defer", "exclude", "full_refresh", "select", "vars"] + ); assert_eq!(build["properties"]["full_refresh"]["type"], "boolean"); + // `defer` is a per-run toggle rather than a descriptor-only setting: the + // run that publishes an environment's state and the run that defers to + // it are two invocations of ONE script. + assert_eq!(build["properties"]["defer"]["type"], "boolean"); // Defaults come from the descriptor, so an untouched run reproduces it. assert_eq!( build["properties"]["select"]["default"], diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 843a83a310..30c6047255 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -29,10 +29,10 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, the warehouse named as the - /// workspace configures it. The scheme names the producer, the path stays - /// the relation — see `windmill_types::AssetKind::Dbt`. + /// A warehouse relation, `dbt:////`, the warehouse + /// named as the workspace configures it. The scheme names the namespace dbt + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } @@ -288,11 +288,13 @@ pub struct RetrySpec { } // `// materialize [manual] [append] [key=] [history] [track=]` -// — declares that this script produces a *managed* materialization of `` -// (a `ducklake://` table). By default the runtime generates the write DDL around +// — declares that this script produces ``. A `ducklake://` table is +// materialized *managed* by default: the runtime generates the write DDL around // the script's single trailing `SELECT` and owns idempotency, partition-state // and snapshot capture. `manual` is the escape hatch: the script writes its own -// DDL and the runtime only records state (track-only). The reconciliation +// DDL and the runtime only records state (track-only) — and it is the only mode a +// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy +// enforces that; see docs/dbt-runtime.md). The reconciliation // strategy options apply to managed mode: none → DELETE-by-partition + INSERT // (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). @@ -804,6 +806,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String { ) } +/// Whether a `dbt://` path names a whole relation, `//`. +/// +/// Every producer spells one that way — the manifest ingest derives it from +/// `relation_name`, a `// materialize` target is checked against it — so anything +/// else can be produced by nothing and read by nothing. Both sides of the deploy +/// ask here rather than counting segments themselves: a subscription and a write +/// that disagreed on the shape would refuse and accept the same string. +pub fn is_full_relation_path(path: &str) -> bool { + let mut segments = path.split('/'); + segments.clone().count() == 3 && !segments.any(str::is_empty) +} + /// A doubled delimiter inside a quoted identifier is that delimiter, literally — /// the same rule the worker's `split_relation` applies to `relation_name`. Both /// have to decode it or one spelling of a table becomes two graph nodes: the dbt @@ -1740,10 +1754,7 @@ mod pipeline_annotation_tests { // just stop being the same node and the cross-boundary cascade never fires. #[test] fn table_paths_from_every_spelling_canonicalize_to_one_key() { - let canonical = Some(( - AssetKind::Dbt, - Cow::Owned("main/analytics/orders".into()), - )); + let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into()))); for spelling in [ // Hand-written annotation. "dbt://main/analytics/orders", @@ -1765,6 +1776,17 @@ mod pipeline_annotation_tests { } } + /// The shape both halves of the deploy check against: a subscription and a + /// write that disagreed on it would refuse and accept the same string. + #[test] + fn a_whole_relation_is_three_non_empty_segments() { + assert!(is_full_relation_path("main/analytics/orders")); + assert!(is_full_relation_path("main/archive.sales/orders")); + for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] { + assert!(!is_full_relation_path(partial), "{partial} is not a relation"); + } + } + // A relation that overrode its database carries `.` in // one segment, and each half can be quoted independently. Stripping only // the outer pair leaves a key the manifest ingest never produces, so the @@ -1791,10 +1813,7 @@ mod pipeline_annotation_tests { // database qualifier. assert_eq!( parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), - Some(( - AssetKind::Dbt, - Cow::Owned("main/sales.v2/orders".into()) - )) + Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into()))) ); } @@ -1813,14 +1832,8 @@ mod pipeline_annotation_tests { "dbt://main/analytics/\"order\"\"s\"", "main/analytics/order\"s", ), - ( - "dbt://main/`da``ta`/`orders`", - "main/da`ta/orders", - ), - ( - "dbt://main/[my]]schema]/[orders]", - "main/my]schema/orders", - ), + ("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"), + ("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"), // And in one half of a database-qualified segment. ( "dbt://main/\"arch\"\"ive\".\"sales\"/orders", @@ -1839,8 +1852,14 @@ mod pipeline_annotation_tests { // apart. A lone delimiter treated as opening a quote would be dropped — // `sa"les` filed as `sales` — and the two derivations would split. for (decoded, spelled) in [ - ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), - ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/sa\"les/orders", + "dbt://main/\"sa\"\"les\"/orders", + ), + ( + "dbt://main/analytics/order\"s", + "dbt://main/analytics/\"order\"\"s\"", + ), ( "dbt://main/arch\"ive.sales/orders", "dbt://main/\"arch\"\"ive\".\"sales\"/orders", diff --git a/backend/src/main.rs b/backend/src/main.rs index f84812151d..316ed099f2 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -46,7 +46,8 @@ use windmill_common::{ CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, @@ -135,7 +136,8 @@ use crate::monitor::{ reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting, reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, - reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting, + reload_extra_pip_index_url_setting, reload_http_route_default_allowed_origins_setting, + reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, @@ -409,6 +411,13 @@ struct HubResourceTypeRaw { /// Absent from hubs predating the column, and from caches written before it. #[serde(default)] pub format_extension: Option, + /// Doubly optional, so a hub predating the field (no key) is told apart from a type the + /// hub leaves unnamed (null). + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } @@ -432,6 +441,14 @@ pub struct HubResourceType { skip_serializing_if = "Option::is_none" )] pub format_extension: Option>, + /// Doubly optional like `format_extension`: a cache written before the field leaves the + /// stored name alone, while a null from the hub clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub display_name: Option>, } const HUB_RT_CACHE_FILE: &str = "resource_types.json"; @@ -479,6 +496,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { app: rt.app, description: rt.description, format_extension: Some(rt.format_extension), + display_name: rt.display_name, }) }) .collect(); @@ -529,8 +547,9 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh Option, Option, bool, + Option, )> = sqlx::query_as( - "SELECT name, schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = 'admins'", + "SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'", ) .fetch_all(db) .await @@ -538,12 +557,23 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh let existing_map: std::collections::HashMap< String, - (Option, Option, Option, bool), + ( + Option, + Option, + Option, + bool, + Option, + ), > = existing_types .into_iter() - .map(|(name, schema, desc, format_extension, is_fileset)| { - (name, (schema, desc, format_extension, is_fileset)) - }) + .map( + |(name, schema, desc, format_extension, is_fileset, display_name)| { + ( + name, + (schema, desc, format_extension, is_fileset, display_name), + ) + }, + ) .collect(); let mut synced_count = 0; @@ -551,8 +581,9 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh for rt in cached_types { let existing = existing_map.get(&rt.name); - let is_fileset = existing.map(|(_, _, _, f)| *f).unwrap_or(false); - let stored_extension = existing.and_then(|(_, _, e, _)| e.clone()); + let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false); + let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone()); + let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone()); // A fileset is a set of files, so it cannot also be one file. Create, update // and the manual sync all reject the pair; this writer would otherwise // persist it onto a same-named local fileset. @@ -568,11 +599,25 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh None => stored_extension.clone(), } }; + // No key in the cache leaves the stored name alone, as for the extension. So does a name + // too long for the column: one bad entry must not fail the upsert and end the sync. + let display_name = match &rt.display_name { + Some(Some(name)) if name.chars().count() > 100 => { + tracing::warn!( + "Ignoring the display_name of resource type {}: longer than 100 characters", + rt.name + ); + stored_display_name.clone() + } + Some(from_cache) => from_cache.clone(), + None => stored_display_name.clone(), + }; - if let Some((existing_schema, existing_desc, _, _)) = existing { + if let Some((existing_schema, existing_desc, _, _, _)) = existing { if existing_schema == &rt.schema && existing_desc == &rt.description && stored_extension == format_extension + && stored_display_name == display_name { skipped_count += 1; continue; @@ -584,16 +629,18 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh // `format_extension` is resolved above rather than coalesced here: a // COALESCE could never clear one, so a hub that dropped an extension // would leave the stale value behind forever. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) - VALUES ('admins', $1, $2, $3, $4, now()) + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, name) DO UPDATE SET schema = EXCLUDED.schema, description = EXCLUDED.description, - format_extension = EXCLUDED.format_extension, edited_at = now()", + format_extension = EXCLUDED.format_extension, + display_name = EXCLUDED.display_name, edited_at = now()", ) .bind(&rt.name) .bind(&rt.schema) .bind(&rt.description) .bind(&format_extension) + .bind(&display_name) .execute(db) .await .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; @@ -1913,6 +1960,17 @@ async fn process_notify_event( ); windmill_api::auth::invalidate_token_from_cache(payload); } + "notify_user_email_change" => { + // `:`, or `*:` from a `password` change, which + // knows the name but no workspace. Workspace ids can't contain ':'. + if let Some(username) = payload.strip_prefix("*:") { + tracing::info!("Superadmin identity change detected, invalidating: {username}"); + windmill_common::users::invalidate_email_cache_for_username(username); + } else if let Some((workspace_id, username)) = payload.split_once(':') { + tracing::info!("User email change detected, invalidating cache: {payload}"); + windmill_common::users::invalidate_email_cache(workspace_id, username); + } + } "notify_app_policy_change" => { // payload is `:`; workspace ids can't contain ':'. if server_mode { @@ -1944,6 +2002,13 @@ async fn process_notify_event( if let Err(e) = reload_license_key(&db.into()).await { tracing::error!("Failed to reload license key: {e:#}"); } + // The worker-group cache override is Enterprise-only, and nothing else + // re-reads the plan for it: the periodic settings pass runs ahead of + // reload_license_key, so it would see the plan this event just replaced. + #[cfg(feature = "parquet")] + if worker_mode { + crate::monitor::reload_cache_object_store_override_with_retry(db).await; + } } DEFAULT_TAGS_PER_WORKSPACE_SETTING => { if let Err(e) = load_tag_per_workspace_enabled(db).await { @@ -2127,6 +2192,11 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload app workspaced route setting"); } } + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING => { + if let Err(e) = reload_http_route_default_allowed_origins_setting(db).await { + tracing::error!(error = %e, "Could not reload http route default allowed origins setting"); + } + } HTTP_ROUTE_WORKSPACED_ROUTE_SETTING => { if let Err(e) = reload_http_route_workspaced_route_setting(db).await { tracing::error!(error = %e, "Could not reload http route workspaced route setting"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index a11ed55882..dc3b9b6e8b 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -106,8 +106,13 @@ use windmill_common::{ use windmill_common::{ client::AuthedClient, global_settings::{ - APP_WORKSPACED_ROUTE_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE, - HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + parse_allowed_origins_setting, APP_WORKSPACED_ROUTE_SETTING, + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + HTTP_ROUTE_WORKSPACED_ROUTE, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + }, + queue_metrics::{ + QueueSample, QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX, QUEUE_DELAY_SAME_HEAD_SECS, + QUEUE_METRIC_HEARTBEAT_SECS, QUEUE_METRIC_STALE_SECS, }, }; #[cfg(feature = "parquet")] @@ -354,7 +359,9 @@ pub async fn initial_load( ) } }); - pass.action(windmill_common::min_version::store_min_keep_alive_version(db)); + pass.action(windmill_common::min_version::store_min_keep_alive_version( + db, + )); pass.setting( windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING, false, @@ -396,6 +403,8 @@ pub async fn initial_load( additional_python_paths: None, pip_local_dependencies: None, native_mode, + // an agent worker never reads its group's config, only its token + object_store_cache_config: None, })); } } @@ -420,6 +429,18 @@ pub async fn initial_load( pass.setting(APP_WORKSPACED_ROUTE_SETTING, false, |v| async move { apply_app_workspaced_route_setting(v) }); + pass.setting( + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + false, + |v| async move { + if let Err(e) = apply_http_route_default_allowed_origins_setting(v) { + tracing::error!( + "Error reloading http route default allowed origins: {:?}", + e + ) + } + }, + ); pass.setting( HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, false, @@ -709,7 +730,6 @@ pub async fn initial_load( pass.run(conn).await; } - pub fn apply_metrics_enabled(value: Option) { if let Some(serde_json::Value::Bool(t)) = value { METRICS_ENABLED.store(t, Ordering::Relaxed) @@ -1066,8 +1086,8 @@ pub fn apply_fork_workspace_tag_append_fork_suffix(value: Option error::Result<()> { - let v = - load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await?; + let v = load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true) + .await?; apply_critical_alert_mute_ui_setting(v); Ok(()) } @@ -1764,6 +1784,23 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), } + let expired_login_links_r: std::result::Result, _> = + // Expired rows stay a day so an open still reports "expired" rather than "invalid". + sqlx::query_scalar( + "DELETE FROM login_link WHERE expiration <= now() - interval '1 day' RETURNING token_hash", + ) + .fetch_all(db) + .await; + + match expired_login_links_r { + Ok(hashes) => { + if !hashes.is_empty() { + tracing::info!("deleted {} expired login links", hashes.len()) + } + } + Err(e) => tracing::error!("Error deleting login links: {}", e.to_string()), + } + let pip_resolution_r = sqlx::query_scalar!( "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash", ) @@ -1875,6 +1912,17 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::info!("deleted {} expired otel trace spans", deleted_spans); } + if let Err(e) = sqlx::query!( + "DELETE FROM ai_shared_artifact + WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + windmill_common::ai_shared_artifact_retention_secs(), + ) + .execute(db) + .await + { + tracing::error!("Error deleting expired shared AI artifacts: {:?}", e); + } + let audit_retention_days = audit_log_retention_days().await; let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; @@ -2732,7 +2780,6 @@ pub async fn reload_timeout_wait_result_setting(conn: &Connection) { .await; } - pub async fn reload_extra_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2823,7 +2870,6 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) { .await; } - pub async fn reload_nuget_config_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2931,7 +2977,6 @@ pub async fn reload_ruby_repos_setting(conn: &Connection) { .await; } - pub async fn reload_workspace_registries_setting(conn: &Connection) { match load_value_from_global_settings_with_conn( conn, @@ -3185,7 +3230,6 @@ pub async fn apply_job_isolation_setting(value: Option) { } } - async fn resolve_license_key_value(conn: &Connection, quiet: bool) -> anyhow::Result { let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true) .await @@ -3480,7 +3524,10 @@ impl<'a> SettingsPass<'a> { // on compile-time defaults until the next full reload. Only the single-query transport // can fail this way; over HTTP the batch already is the per-setting read. if matches!(conn, Connection::Sql(_)) && values.is_empty() && !names.is_empty() { - tracing::warn!("Falling back to per-setting reads for {} settings", names.len()); + tracing::warn!( + "Falling back to per-setting reads for {} settings", + names.len() + ); values = fetch_settings_individually(conn, &names).await; } for (name, http) in &declared { @@ -3872,7 +3919,6 @@ pub fn parse_setting_value( value } - #[cfg(feature = "prometheus")] pub async fn monitor_pool(db: &DB) { if METRICS_ENABLED.load(Ordering::Relaxed) { @@ -4316,6 +4362,41 @@ pub async fn monitor_db( } }; + // Re-check what each git-sync repository's own credential says about its expiry, + // and rotate the ones close to it. Every ~40 min: the values move over days, and + // `should_run` counts iterations in a u8. Spawned rather than joined: the join + // below is cancelled at its deadline, which a long sweep would reach, and a + // rotation cut off between GitLab issuing a token and Windmill storing it + // loses the token family. Detached, only process shutdown can cut it off, + // which a rotation almost never coincides with. The pass's advisory lock + // keeps a slow one from overlapping the next. + let git_credential_maintenance_f = async { + #[cfg(all(feature = "enterprise", feature = "private"))] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { maintain_git_credentials(&db).await }); + } + } + }; + + // Delete the AI session backups older than their workspace's retention. Every ~40 min + // (240 iterations at the default 10 s, the most a u8 `should_run` counts): the retention + // counts in days. Spawned for the same reason as the credential maintenance above, a + // sweep of many sessions outlasting the join's deadline; the sweep's own advisory lock + // keeps one server at a time at it. + let ai_session_retention_f = async { + #[cfg(feature = "parquet")] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn( + async move { windmill_api::sweep_expired_ai_session_backups(&db).await }, + ); + } + } + }; + // run every 2 iterations (~20s at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). // Enterprise feature: the active `// freshness` backstop lives in // windmill-queue's `freshness_watchdog` (`private`); OSS gets a no-op stub. @@ -4369,6 +4450,8 @@ pub async fn monitor_db( export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, git_auto_pull_f, + git_credential_maintenance_f, + ai_session_retention_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, ); @@ -4674,6 +4757,14 @@ pub async fn poll_git_auto_pull(db: &Pool) { { tracing::error!("git auto-pull: advisory unlock failed: {e:#}"); } + + // Backstop for the "Windmill CI tests" checks: retry a failed GitHub create or + // delivery, conclude checks whose tests settled, time out stuck ones, prune old + // rows. Detached and outside the advisory lock: its writes are guarded (claimed + // conclude, greatest-id upsert), it is single-flight, and its GitHub calls must not + // count against the monitor pass's budget. + let db = db.clone(); + tokio::spawn(async move { windmill_git_sync::sweep_ci_test_checks(&db).await }); } #[cfg(feature = "private")] @@ -4691,6 +4782,156 @@ lazy_static::lazy_static! { #[cfg(feature = "private")] const AUTO_PULL_POLL_SLACK_S: i64 = 30; +/// Advisory lock id ensuring only one server replica maintains git credentials at +/// a time (adjacent to GIT_AUTO_PULL_LOCK_ID). +#[cfg(all(feature = "enterprise", feature = "private"))] +const GIT_CREDENTIAL_LOCK_ID: i64 = 737_483_923; + +/// Refresh every git-sync repository's credential status and rotate the ones near +/// expiry, so a token dies visibly (and usually not at all) rather than taking +/// sync down on its expiry date. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials(db: &Pool) { + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + return; + } + + // Transaction-scoped advisory lock, as for the schedule reconcile above: a + // session lock on a pooled connection would ride back into the pool still + // held if the sweep died before unlocking, and wedge the pass on every + // replica until a restart. The transaction only owns the lock; the sweep + // commits each status and each rotated token on its own as it goes. + let mut lock_tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("git credentials: failed to begin lock tx: {e:#}"); + return; + } + }; + // The transaction stays idle while the sweep talks to git hosts, and the + // pool's ten-minute idle-in-transaction timeout would end it, lock included, + // partway through a sweep over enough slow hosts. Lifted for this + // transaction only; it dies with the connection either way. + if let Err(e) = sqlx::query("SET LOCAL idle_in_transaction_session_timeout = 0") + .execute(&mut *lock_tx) + .await + { + tracing::error!("git credentials: failed to lift the idle timeout: {e:#}"); + return; + } + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(GIT_CREDENTIAL_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("git credentials: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + return; + } + + if let Err(e) = maintain_git_credentials_inner(db).await { + tracing::error!("git credentials: maintenance error: {e:#}"); + } + drop(lock_tx); +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials_inner(db: &Pool) -> error::Result<()> { + use windmill_common::workspaces::WorkspaceGitSyncSettings; + + // Same deleted/archived exclusion as the auto-pull poller: a dead workspace's + // settings row survives, and rotating a token for one would be pure damage. + let rows = sqlx::query!( + r#"SELECT ws.workspace_id, ws.git_sync + FROM workspace_settings ws + JOIN workspace w ON w.id = ws.workspace_id + WHERE NOT w.deleted + AND ws.git_sync IS NOT NULL + AND jsonb_typeof(ws.git_sync->'repositories') = 'array'"# + ) + .fetch_all(db) + .await?; + + for row in rows { + let Some(git_sync) = row.git_sync else { + continue; + }; + let settings: WorkspaceGitSyncSettings = match serde_json::from_value(git_sync) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + "git credentials: invalid git_sync settings for workspace {}: {e}", + row.workspace_id + ); + continue; + } + }; + + for repo in settings.repositories.iter() { + let path = &repo.git_repo_resource_path; + // This refreshes and records the status on every repository it looks at, + // rotating only the ones near expiry, so it is the whole maintenance pass + // rather than just the rotation half. + if let Err(e) = windmill_common::git_sync_ee::rotate_git_credential_if_due( + db, + &row.workspace_id, + path, + ) + .await + { + tracing::error!( + "git credentials: maintenance failed for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + + // A repository that wants webhook delivery but holds no hook never + // gets one otherwise: the reconcile runs on a settings save, so a + // credential that was unusable when the hook should have been created + // would leave it missing until an admin saved again. Checking stored + // state costs nothing, and only the repositories actually missing a + // hook reach the host. + use windmill_common::workspaces::AutoPullMode; + // Also when a hook exists but carries a warning: a save during a GitLab + // outage keeps the hook and records why it could not be confirmed, and + // that warning is only cleared by a reconcile that confirms it again. + // Only repositories with a checked credential of their own: for + // everything else a settings save stays the one place hooks are + // reconciled, so an App repository or a plain remote is never touched + // here, and a recorded delivery mode nobody saved is never normalized. + let needs_hook = repo.credential.is_some() + && repo.auto_pull.as_ref().is_some_and(|a| { + a.enabled + && matches!(a.mode, AutoPullMode::Auto | AutoPullMode::Webhook) + && (a.webhook_id.is_none() || a.webhook_error.is_some()) + }); + if needs_hook { + let mut repo = repo.clone(); + if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook( + db, + &row.workspace_id, + &mut repo, + ) + .await + { + tracing::warn!( + "git credentials: could not reconcile the webhook for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + } + } + } + Ok(()) +} + #[cfg(feature = "private")] async fn poll_git_auto_pull_inner(db: &Pool) -> error::Result<()> { use windmill_common::workspaces::{AutoPullMode, WorkspaceGitSyncSettings}; @@ -4936,155 +5177,303 @@ async fn vacuuming_tables(db: &Pool) -> error::Result<()> { Ok(()) } -pub async fn expose_queue_metrics(db: &Pool) { - let last_check = sqlx::query_scalar!( - "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1" - ) - .fetch_optional(db) - .await - .unwrap_or(Some(chrono::Utc::now())); +/// Shortest spacing between two stored samples of the same queue metric, so a tag whose +/// value moves on every monitor round still writes at most one row per interval. Also how +/// often each server samples the queue when no Prometheus or OTel gauge needs it sooner. +const QUEUE_METRIC_MIN_INTERVAL_SECS: f64 = 25.0; +/// A held delay hovers while the head keeps changing, so an exact-value comparison would rarely +/// dedup it. Only a move the chart would actually render is stored. +const QUEUE_DELAY_TOLERANCE: f64 = 0.1; - let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); - let save_metrics = last_check - .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25)) - .unwrap_or(true); +/// Append the queue metrics the drawer at `GET /workers/queue_metrics_series` charts, skipping +/// any sample that repeats what is already stored. +/// +/// Only tags with a backlog appear in `queue_stats`, and an arbitrary `?tag=` nobody serves +/// stays backlogged forever, so writing every round would repeat the same pair of rows for +/// the whole 14-day retention. Each metric is written when the value it draws moves, once per +/// heartbeat while it holds, and once more (as a zero) when the tag drains. Gaps therefore +/// mean "unchanged since the last row", which is what the chart interpolates. A delay whose +/// head job stays put is stored as that job's wait start, which the chart draws climbing, so it +/// never moves away from what is stored either. +async fn save_queue_metrics( + db: &Pool, + queue_stats: &std::collections::HashMap, +) { + let sampled_ids = queue_stats + .keys() + .flat_map(|tag| { + [ + format!("{QUEUE_COUNT_PREFIX}{tag}"), + format!("{QUEUE_DELAY_PREFIX}{tag}"), + ] + }) + .collect::>(); - if metrics_enabled || save_metrics || OTEL_METRICS_ENABLED.load(Ordering::Relaxed) { - let queue_counts = windmill_common::queue::get_queue_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - (*QUEUE_COUNT).with_label_values(&[q]).set(0); - } - } + // Last stored sample of every metric that either has a backlog now or was written + // recently enough to still be believed backlogged. Bounding the lookup by the stale window + // keeps it cheap at any `metrics` size; a per-id `ORDER BY created_at DESC LIMIT 1` does + // not, since the planner may serve it from `metrics_sort_idx` and walk the whole table. + let last_samples = match sqlx::query!( + "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\", + EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\", + EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\" + FROM unnest($1::text[]) AS c(id) + FULL JOIN ( + SELECT DISTINCT ON (id) id, value, created_at + FROM metrics + WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2) + ORDER BY id, created_at DESC + ) r ON r.id = c.id", + &sampled_ids[..], + QUEUE_METRIC_STALE_SECS, + ) + .fetch_all(db) + .await + { + Ok(rows) => rows, + Err(e) => { + tracing::error!("Failed to read last queue metrics samples: {e:#}"); + return; } + }; - let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let mut ids = vec![]; + let mut values = vec![]; + // The wait start of the head of each held delay, whose value the INSERT computes. + let mut held_heads: Vec> = vec![]; + for row in last_samples { + let Some((prefix, tag)) = [QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX] + .into_iter() + .find_map(|p| row.id.strip_prefix(p).map(|tag| (p, tag))) + else { + continue; + }; + // A stored value that cannot be read cannot be compared, so the next reading is kept. + let last = row + .value + .as_ref() + .and_then(QueueSample::parse) + .zip(row.at) + .zip(row.age) + .map(|((sample, at), age)| (sample, at, age)); + let stat = queue_stats.get(tag); + let current = stat.map(|stat| { + if prefix == QUEUE_COUNT_PREFIX { + stat.count as f64 + } else { + stat.delay + } + }); - if otel_enabled { - for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - otel_set_queue_count(q, 0); + let next_delay = stat + .filter(|_| prefix == QUEUE_DELAY_PREFIX) + .map(|stat| delay_sample(last.map(|(sample, at, _)| sample.head_since(at)), stat)); + let drawn_now = last.map(|(sample, at, age)| (sample.value_at(at + age), age)); + let redraws = last + .zip(next_delay) + .is_some_and(|((sample, ..), next)| redraws(sample, next)); + if should_store(prefix, drawn_now, current, redraws) { + let (value, held_head) = match (stat, next_delay) { + (None, _) => (serde_json::json!(0), None), + (Some(stat), None) => (serde_json::json!(stat.count), None), + (Some(stat), Some(QueueSample::Held(_))) => { + (serde_json::Value::Null, Some(stat.head_since)) } - } - } - - #[allow(unused_mut)] - let mut tags_to_watch = vec![]; - #[allow(unused_mut)] - let mut otel_tags_to_watch = vec![]; - for q in queue_counts { - let count = q.1; - let tag = q.0; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); - metric.set(count as i64); - tags_to_watch.push(tag.to_string()); - } - - if otel_enabled { - otel_tags_to_watch.push(tag.to_string()); - } - otel_set_queue_count(&tag, count as i64); - - // save queue_count and delay metrics per tag - if save_metrics { - sqlx::query!( - "INSERT INTO metrics (id, value) VALUES ($1, $2)", - format!("queue_count_{}", tag), - serde_json::json!(count) - ) - .execute(db) - .await - .ok(); - if count > 0 { - sqlx::query!( - "INSERT INTO metrics (id, value) - VALUES ($1, to_jsonb(( - SELECT EXTRACT(EPOCH FROM now() - scheduled_for) - FROM v2_job_queue - WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval - ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1 - )))", - format!("queue_delay_{}", tag), - tag - ) - .execute(db) - .await - .ok(); - } - } - } - if metrics_enabled { - let mut w = QUEUE_COUNT_TAGS.write().await; - *w = tags_to_watch; - } - if otel_enabled { - let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; - *w = otel_tags_to_watch; - } - - // Single DB query for running counts, shared by Prometheus and OTel - let otel_running = otel_enabled; - #[cfg(feature = "prometheus")] - let need_running_counts = metrics_enabled || otel_running; - #[cfg(not(feature = "prometheus"))] - let need_running_counts = otel_running; - - if need_running_counts { - let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); - } - } - } - - if otel_running { - for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - otel_set_queue_running_count(q, 0); - } - } - } - - #[allow(unused_mut, unused_variables)] - let mut running_tags_to_watch: Vec = vec![]; - #[allow(unused_mut, unused_variables)] - let mut otel_running_tags_to_watch: Vec = vec![]; - for (tag, count) in &queue_running_counts { - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); - metric.set(*count as i64); - running_tags_to_watch.push(tag.to_string()); - } - - if otel_running { - otel_set_queue_running_count(tag, *count as i64); - otel_running_tags_to_watch.push(tag.to_string()); - } - } - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = running_tags_to_watch; - } - if otel_running { - let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = otel_running_tags_to_watch; - } + (Some(_), Some(climbing)) => (climbing.to_json(), None), + }; + ids.push(row.id); + values.push(value); + held_heads.push(held_head); } } + if ids.is_empty() { + return; + } + // A held delay is computed from this statement's `now()`, the row's `created_at` too, so + // `created_at - value` is exactly its head's wait start. That is how the next sample tells + // whether the same job is still at the head, within `QUEUE_DELAY_SAME_HEAD_SECS`, which the + // time between reading the queue and this INSERT could otherwise exceed on a busy database. + if let Err(e) = sqlx::query!( + "INSERT INTO metrics (id, value) + SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value) + FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + &ids[..], + &values[..], + &held_heads[..] as &[Option], + ) + .execute(db) + .await + { + tracing::error!("Failed to save queue metrics: {e:#}"); + } +} + +/// What to store for a delay reading, given when the head job of the last stored sample started +/// waiting. The same job still at the head keeps the delay climbing from its wait start, which +/// the chart draws exactly. A head that changed means a moving queue, whose delay hovers and is +/// held; so is a first sample, which cannot tell yet and must not draw a climb that never was. +fn delay_sample( + last_head_since: Option, + stat: &windmill_common::queue::QueueStat, +) -> QueueSample { + match last_head_since { + Some(since) if (since - stat.head_since).abs() < QUEUE_DELAY_SAME_HEAD_SECS => { + QueueSample::Climbing { since: stat.head_since } + } + _ => QueueSample::Held(stat.delay), + } +} + +/// Whether the next delay sample is drawn differently from the last one even at the same value: +/// a climb whose head left would otherwise go on climbing from the old head, and a held delay +/// whose head stayed would stay flat while the wait grows. +fn redraws(last: QueueSample, next: QueueSample) -> bool { + matches!(last, QueueSample::Climbing { .. }) != matches!(next, QueueSample::Climbing { .. }) +} + +/// Whether a reading deserves a row of its own, given the last one stored for that metric: +/// the value it draws now and how many seconds ago it was written. `current` is `None` once +/// the tag has no backlog left; `redraws` is set when the reading must be drawn differently. +fn should_store( + prefix: &str, + last: Option<(f64, f64)>, + current: Option, + redraws: bool, +) -> bool { + let Some((last_value, age)) = last else { + // Nothing comparable within the lookback window: a tag that just backed up needs a + // first sample, one that was already gone needs nothing. + return current.is_some(); + }; + let Some(current) = current else { + // The tag drained. One zero pins where the line drops; after that the metric matches + // and goes quiet, then falls out of the lookback window entirely. + return last_value != 0.0; + }; + if age >= QUEUE_METRIC_HEARTBEAT_SECS { + return true; + } + age >= QUEUE_METRIC_MIN_INTERVAL_SECS + && (redraws + || if prefix == QUEUE_COUNT_PREFIX { + last_value != current + } else { + (current - last_value).abs() > last_value.abs() * QUEUE_DELAY_TOLERANCE + }) +} + +#[cfg(test)] +mod queue_metric_sampling { + use super::*; + + const RECENT: f64 = QUEUE_METRIC_MIN_INTERVAL_SECS + 1.0; + + #[test] + fn a_holding_backlog_writes_only_on_the_heartbeat() { + let held = Some((3.0, RECENT)); + assert!(!should_store(QUEUE_COUNT_PREFIX, held, Some(3.0), false)); + let due = Some((3.0, QUEUE_METRIC_HEARTBEAT_SECS)); + assert!(should_store(QUEUE_COUNT_PREFIX, due, Some(3.0), false)); + // A held delay hovers, so only a move past the tolerance counts as a change. + let delay = Some((100.0, RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, delay, Some(105.0), false)); + assert!(should_store(QUEUE_DELAY_PREFIX, delay, Some(120.0), false)); + } + + #[test] + fn a_drained_tag_writes_one_zero_then_stops() { + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + None, + false + )); + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((0.0, RECENT)), + None, + false + )); + // Including once the heartbeat is due: a tag that is gone stays silent. + let gone = Some((0.0, QUEUE_METRIC_STALE_SECS)); + assert!(!should_store(QUEUE_COUNT_PREFIX, gone, None, false)); + assert!(!should_store(QUEUE_COUNT_PREFIX, None, None, false)); + } + + #[test] + fn a_change_waits_for_the_minimum_interval() { + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, 1.0)), + Some(9.0), + false + )); + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + Some(9.0), + false + )); + // A tag that has just backed up is recorded at once. + assert!(should_store(QUEUE_COUNT_PREFIX, None, Some(9.0), false)); + } + + #[test] + fn a_delay_climbs_while_the_same_job_stays_at_the_head() { + let stat = windmill_common::queue::QueueStat { count: 3, delay: 330.0, head_since: 1000.0 }; + // A held sample written at 1320 saw the same head: it switches to climbing at once, + // although the delay has not moved past the tolerance yet. + let first = QueueSample::Held(320.0); + let climbing = delay_sample(Some(first.head_since(1320.0)), &stat); + assert_eq!(climbing, QueueSample::Climbing { since: 1000.0 }); + assert!(redraws(first, climbing)); + let drawn = Some((first.value_at(1330.0), RECENT)); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(stat.delay), + true + )); + // Stored climbing, it draws the delay exactly: nothing more until the heartbeat. + let drawn = Some((climbing.value_at(1600.0), RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, drawn, Some(600.0), false)); + assert_eq!(delay_sample(None, &stat), QueueSample::Held(330.0)); + } + + #[test] + fn a_climb_whose_head_left_is_held_even_within_the_tolerance() { + // The head waiting since 0 left at 3600 for one queued at 100: 3500s is within 10% of + // the 3600s the climb draws, but kept, the climb would go on from the old head. + let moved = + windmill_common::queue::QueueStat { count: 2, delay: 3500.0, head_since: 100.0 }; + let climbing = QueueSample::Climbing { since: 0.0 }; + let next = delay_sample(Some(climbing.head_since(3000.0)), &moved); + assert_eq!(next, QueueSample::Held(3500.0)); + assert!(redraws(climbing, next)); + let drawn = Some((climbing.value_at(3600.0), RECENT)); + assert!(!should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + false + )); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + true + )); + } +} + +/// When this server last sampled the queue into `metrics`, in Unix milliseconds. It only paces +/// how often the queue is scanned for that; whether a sample earns a row is decided from what +/// is already stored. Servers sampling in the same instant can each write it, and the duplicate +/// draws the same. +static LAST_QUEUE_SAMPLE_MS: std::sync::atomic::AtomicI64 = std::sync::atomic::AtomicI64::new(0); + +pub async fn expose_queue_metrics(db: &Pool) { // clean queue metrics older than 14 days sqlx::query!( "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'" @@ -5092,6 +5481,131 @@ pub async fn expose_queue_metrics(db: &Pool) { .execute(db) .await .ok(); + + let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); + let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let now_ms = chrono::Utc::now().timestamp_millis(); + let save_metrics = now_ms - LAST_QUEUE_SAMPLE_MS.load(Ordering::Relaxed) + >= (QUEUE_METRIC_MIN_INTERVAL_SECS * 1000.0) as i64; + if !(metrics_enabled || otel_enabled || save_metrics) { + return; + } + + // Single DB query for running counts, shared by Prometheus and OTel. It runs ahead of the + // backlog read below, which gives up on the rest of the round when it fails. + let otel_running = otel_enabled; + #[cfg(feature = "prometheus")] + let need_running_counts = metrics_enabled || otel_running; + #[cfg(not(feature = "prometheus"))] + let need_running_counts = otel_running; + + if need_running_counts { + let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_running { + for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + otel_set_queue_running_count(q, 0); + } + } + } + + #[allow(unused_mut, unused_variables)] + let mut running_tags_to_watch: Vec = vec![]; + #[allow(unused_mut, unused_variables)] + let mut otel_running_tags_to_watch: Vec = vec![]; + for (tag, count) in &queue_running_counts { + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); + metric.set(*count as i64); + running_tags_to_watch.push(tag.to_string()); + } + + if otel_running { + otel_set_queue_running_count(tag, *count as i64); + otel_running_tags_to_watch.push(tag.to_string()); + } + } + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = running_tags_to_watch; + } + if otel_running { + let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = otel_running_tags_to_watch; + } + } + + let queue_stats = match windmill_common::queue::get_queue_stats(db).await { + Ok(queue_stats) => queue_stats, + Err(e) => { + tracing::error!("Failed to read queue stats: {e:#}"); + return; + } + }; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + (*QUEUE_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_enabled { + for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + otel_set_queue_count(q, 0); + } + } + } + + #[allow(unused_mut)] + let mut tags_to_watch = vec![]; + #[allow(unused_mut)] + let mut otel_tags_to_watch = vec![]; + for (tag, stat) in queue_stats.iter() { + let count = stat.count; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_COUNT).with_label_values(&[tag]); + metric.set(count as i64); + tags_to_watch.push(tag.to_string()); + } + + if otel_enabled { + otel_tags_to_watch.push(tag.to_string()); + } + otel_set_queue_count(tag, count as i64); + } + + if save_metrics { + LAST_QUEUE_SAMPLE_MS.store(now_ms, Ordering::Relaxed); + save_queue_metrics(db, &queue_stats).await; + } + + if metrics_enabled { + let mut w = QUEUE_COUNT_TAGS.write().await; + *w = tags_to_watch; + } + if otel_enabled { + let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; + *w = otel_tags_to_watch; + } } pub async fn reload_smtp_config(db: &Pool) { @@ -5126,6 +5640,7 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b .dedicated_workers .as_ref() .is_some_and(|dws| !dws.is_empty()); + if **wc != config || has_dedicated { if kill_if_change { if has_dedicated @@ -5173,6 +5688,37 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b store_pull_query(&config).await; WORKER_CONFIG.store(std::sync::Arc::new(config)); } + + // After the store, so a retry that wakes mid-build reads the config being applied + // rather than the one it replaced. Unconditional rather than gated on the value + // changing, so that a pass triggered by anything else — a license-plan change, most + // of all — still re-evaluates the entitlement. + #[cfg(feature = "parquet")] + reload_cache_object_store_override_with_retry(db).await; + } +} + +/// Apply this worker group's dependency-cache object store, retrying once shortly after a build +/// that failed for a reason that may pass — the periodic settings reload behind it is 12h apart, +/// which is a long time for a whole group to cache nothing but locally. +#[cfg(feature = "parquet")] +pub async fn reload_cache_object_store_override_with_retry(db: &DB) { + let settings = WORKER_CONFIG.load().object_store_cache_config.clone(); + if matches!( + windmill_object_store::reload_cache_object_store_override(db, settings).await, + ObjectStoreReload::Later + ) { + let db = db.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(10)).await; + if windmill_object_store::cache_object_store_override_failed().await { + // Re-read rather than reuse: the group config may have changed while we slept, + // and installing the settings this retry was born with would pin the worker to a + // store the group no longer asks for. + let settings = WORKER_CONFIG.load().object_store_cache_config.clone(); + windmill_object_store::reload_cache_object_store_override(&db, settings).await; + } + }); } } @@ -5651,7 +6197,10 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n /// Force-complete a zombie job that handle_job_error failed to complete. /// This is a minimal fallback: it inserts a failed completed job and deletes /// from the queue in a single transaction, without schedule pushing or -/// error handler logic that could cause the completion to fail. +/// error handler logic. The one thing it keeps is the WAC parent notification, +/// deliberately inside the transaction: if that fails, the whole completion +/// rolls back and the job waits for the next sweep, which is cheaper than a +/// parent parked for its full suspend window and a task run twice. async fn force_complete_zombie_job( db: &Pool, job_id: &Uuid, @@ -5673,14 +6222,18 @@ async fn force_complete_zombie_job( "Zombie job {job_id} was not completed by handle_job_error, force-completing it" ); + // Same `{"error": ...}` shape as every other failed job's result, so a WAC + // parent's failure record reads the name and message like any task failure. let error_value = serde_json::json!({ - "message": error_message, - "name": "ExecutionErr", + "error": { + "message": error_message, + "name": "ExecutionErr", + } }); let mut tx = db.begin().await?; - sqlx::query!( + let duration_ms = sqlx::query_scalar!( "INSERT INTO v2_job_completed (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker) SELECT q.workspace_id, q.id, q.started_at, @@ -5689,19 +6242,50 @@ async fn force_complete_zombie_job( FROM v2_job_queue q LEFT JOIN v2_job_runtime r ON r.id = q.id WHERE q.id = $1 - ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb + RETURNING duration_ms AS \"duration_ms!\"", job_id, error_value, ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + // A WAC parent parked on this job must learn of the failure here too, or it + // waits out its whole suspend window and runs the task again. + let mut wac_parent_ready = false; + if let Some(duration_ms) = duration_ms { + let parent = sqlx::query!( + "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1", + job_id + ) + .fetch_optional(&mut *tx) + .await?; + if let Some(parent_job) = parent + .filter(|j| j.flow_step_id.is_none()) + .and_then(|j| j.parent_job) + { + wac_parent_ready = windmill_common::wac::record_child_completion( + &mut tx, + &parent_job, + job_id, + false, + duration_ms, + &error_value.to_string(), + ) + .await?; + } + } + sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job_id) .execute(&mut *tx) .await?; tx.commit().await?; + if wac_parent_ready { + windmill_common::wac::WAC_SUSPEND_READY.store(true, Ordering::Relaxed); + } + tracing::info!("Force-completed zombie job {job_id}"); Ok(()) } @@ -6523,6 +7107,34 @@ pub fn apply_app_workspaced_route_setting(app_workspaced_route: Option error::Result<()> { + let v = + load_value_from_global_settings(conn, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING).await?; + apply_http_route_default_allowed_origins_setting(v) +} + +pub fn apply_http_route_default_allowed_origins_setting( + value: Option, +) -> error::Result<()> { + // A bad value leaves whatever is already loaded in place rather than + // reverting to no restriction. On the boot path that is still the empty + // default, so what keeps a stored typo from widening CORS instance-wide is + // write-time validation, not this. + let origins = match parse_allowed_origins_setting(value.as_ref()) { + Ok(origins) => origins, + Err(err) => { + tracing::error!( + "Invalid {} setting, keeping the previous value: {err:#}", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ); + return Ok(()); + } + }; + + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.store(std::sync::Arc::new(origins)); + Ok(()) +} + pub async fn reload_http_route_workspaced_route_setting(conn: &DB) -> error::Result<()> { let v = load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?; apply_http_route_workspaced_route_setting(conn, v).await @@ -6588,7 +7200,6 @@ pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<( Ok(()) } - pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { let v = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; apply_jwt_secret_setting(db, v).await diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 61f8a66f85..1e27ee1ce4 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -40,6 +40,8 @@ agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklis ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) +ai_shared_artifact: id(uuid), workspace_id(char), artifact_id(char), email(char), created_by(char), name(char), kind(char), version(int), content(text), shared_at(ts) + FK: (workspace_id) -> workspace(id) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) @@ -70,11 +72,15 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) -dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) +dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) +dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) +dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char), id(bigint) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) + FK: (workspace_id) -> workspace(id) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) FK: (workspace_id) -> workspace(id) @@ -84,7 +90,7 @@ debounce_key: key(char), job_id(uuid), previous_job_id(uuid), first_started_at(t debounce_stale_data: job_id(uuid), to_relock(text[]) debouncing_settings: hash(bigint), debounce_key(char), debounce_delay_s(int), max_total_debouncing_time(int), max_total_debounces_amount(int), debounce_args_to_accumulate(text[]) dependency_map: workspace_id(char), importer_path(char), importer_kind(importer_kind), imported_path(char), importer_node_id(char) -deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid) +deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid), id(bigint) FK: (workspace_id) -> workspace(id) draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at(ts) FK: (workspace_id) -> workspace(id) @@ -109,6 +115,11 @@ folder: name(char), workspace_id(char), display_name(char), owners(char), extra_ folder_permission_history: id(bigint), workspace_id(char), folder_name(char), changed_by(char), changed_at(ts), change_type(char), affected(char) FK: (workspace_id, folder_name) -> folder(workspace_id, name) gcp_trigger: gcp_resource_path(char), topic_id(char), subscription_id(char), delivery_type(delivery_mode), delivery_config(jsonb), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), subscription_mode(gcp_subscription_mode), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), auto_acknowledge_msg(bool), ack_deadline(int), mode(trigger_mode), labels(text[]) +git_sync_ci_test_check: workspace_id(char), head_sha(char), head_ref(char), poster_workspace_id(char), repo_url(text), repo_resource_path(char), check_run_id(bigint), created_at(timestamptz), concluded(bool), conclusion(text), concluded_at(timestamptz), github_posted(bool) + FK: (workspace_id) -> workspace(id) + FK: (poster_workspace_id) -> workspace(id) +git_sync_synced_head: workspace_id(char), repo_resource_path(char), branch(char), sha(char), source(char), job_id(uuid), synced_at(timestamptz), tests_dispatched_at(timestamptz), ci_test_job_ids(uuid[]) + FK: (workspace_id) -> workspace(id) global_settings: name(char), value(jsonb), updated_at(ts) guest_activity: email(char), workspace_id(char), day(date), last_seen_at(timestamptz), jwt_entry(bool) group_: workspace_id(char), name(char), summary(text), extra_perms(jsonb) @@ -116,7 +127,7 @@ group_: workspace_id(char), name(char), summary(text), extra_perms(jsonb) group_permission_history: id(bigint), workspace_id(char), group_name(char), changed_by(char), changed_at(ts), change_type(char), member_affected(char) FK: (workspace_id, group_name) -> group_(workspace_id, name) healthchecks: id(bigint), check_type(text), healthy(bool), created_at(ts) -http_trigger: path(char), route_path(char), route_path_key(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), authentication_method(authentication_method), http_method(http_method), static_asset_config(jsonb), is_static_website(bool), workspaced_route(bool), wrap_body(bool), raw_string(bool), authentication_resource_path(char), summary(char), description(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), request_type(request_type), mode(trigger_mode), labels(text[]) +http_trigger: path(char), route_path(char), route_path_key(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), authentication_method(authentication_method), http_method(http_method), static_asset_config(jsonb), is_static_website(bool), workspaced_route(bool), wrap_body(bool), raw_string(bool), allowed_origins(text[]), authentication_resource_path(char), summary(char), description(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), request_type(request_type), mode(trigger_mode), labels(text[]) input: id(uuid), workspace_id(char), runnable_id(char), runnable_type(runnable_type), name(text), args(jsonb), created_at(ts), created_by(char), is_public(bool) FK: (workspace_id) -> workspace(id) instance_group: name(char), summary(char), id(char), scim_display_name(char), external_id(char) @@ -143,9 +154,9 @@ mcp_oauth_refresh_token: id(bigint), refresh_token(char), access_token_hash(char mcp_oauth_server_client: client_id(char), client_name(char), redirect_uris(text[]), created_at(ts) mcp_oauth_server_code: code(char), client_id(char), user_email(char), workspace_id(char), scopes(text[]), redirect_uri(text), code_challenge(char), code_challenge_method(char), created_at(ts), expires_at(ts) FK: (client_id) -> mcp_oauth_server_client(client_id) -metrics: id(char), value(jsonb), created_at(ts) +metrics: id(char), value(jsonb), created_at(ts), row_id(bigint) mqtt_trigger: mqtt_resource_path(char), subscribe_topics(jsonb[]), client_version(mqtt_client_version), v5_config(jsonb), v3_config(jsonb), client_id(char), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) -native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts) +native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts), enabled(bool) FK: (workspace_id) -> workspace(id) nats_trigger: path(char), nats_resource_path(char), subjects(char), stream_name(char), consumer_name(char), use_jetstream(bool), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) FK: (workspace_id) -> workspace(id) @@ -162,7 +173,7 @@ raw_app: path(char), version(int), workspace_id(char), summary(char), edited_at( FK: (workspace_id) -> workspace(id) resource: workspace_id(char), path(char), value(jsonb), description(text), resource_type(char), extra_perms(jsonb), edited_at(ts), created_by(char), labels(text[]) FK: (workspace_id) -> workspace(id) -resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool) +resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool), display_name(char) FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) @@ -221,9 +232,9 @@ workspace_key: workspace_id(char), kind(workspace_key_kind), key(char) FK: (workspace_id) -> workspace(id) workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_groups(text[]), bypass_users(text[]), created_at(ts) FK: (workspace_id) -> workspace(id) -workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char) +workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint), runnable_is_agent(bool) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index 791556b035..4d6fffa346 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -293,6 +293,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> let lfs_config = json!({ "type": "FilesystemStorage", "root_path": storage_root, + "volume_storage": "primary", "public_resource": null, "advanced_permissions": null }); @@ -306,7 +307,11 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> .await?; // 2. Pre-populate the volume with a file - let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); std::fs::create_dir_all(&vol_dir)?; std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; @@ -343,6 +348,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> // 4. GET /file/* — download the existing file let resp = http .get(format!("{vol_base}/file/hello.txt")) + .query(&[("worker_name", "test-worker-1")]) .send() .await?; assert!( @@ -360,6 +366,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> // 5. PUT /file/* — upload a new file let resp = http .put(format!("{vol_base}/file/output.txt")) + .query(&[("worker_name", "test-worker-1")]) .body(b"written by agent worker".to_vec()) .send() .await?; @@ -432,7 +439,8 @@ async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow: "type": "FilesystemStorage", "root_path": storage_root, "public_resource": null, - "advanced_permissions": null + "advanced_permissions": null, + "volume_storage": "primary" }); sqlx::query!( @@ -444,20 +452,24 @@ async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow: .await?; // 2. Pre-populate the volume with a file - let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); std::fs::create_dir_all(&vol_dir)?; std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; // 3. Push the job, then run worker with HTTP connection (bun tag) - let code = r#"// volume: test-vol /tmp/data + let code = r#"// volume: test-vol data import { readFileSync, writeFileSync, existsSync } from "fs"; export function main() { - const content = readFileSync("/tmp/data/hello.txt", "utf-8"); - writeFileSync("/tmp/data/output.txt", "written by agent worker"); + const content = readFileSync("data/hello.txt", "utf-8"); + writeFileSync("data/output.txt", "written by agent worker"); return { read_content: content, - output_exists: existsSync("/tmp/data/output.txt"), + output_exists: existsSync("data/output.txt"), }; }"#; @@ -528,6 +540,7 @@ async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result< let lfs_config = json!({ "type": "FilesystemStorage", "root_path": storage_root, + "volume_storage": "primary", "public_resource": null, "advanced_permissions": null }); diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs new file mode 100644 index 0000000000..6b33a4141e --- /dev/null +++ b/backend/tests/ai_sessions.rs @@ -0,0 +1,1582 @@ +//! The AI session backup routes (`/w/{w}/ai/sessions/*`): a browser pushes pieces of its +//! sessions into the workspace's object storage and pulls them back whole. Pinned against a +//! FilesystemStorage LFS so the test needs no object store, which also lets it read what +//! landed on disk: the objects must be ciphertext, since bucket credentials are shared far +//! more widely than a user's transcripts. +#![cfg(all(feature = "private", feature = "parquet"))] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_common::utils::calculate_hash; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +async fn configure_primary_lfs(db: &Pool, root_path: &str) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + json!({ + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": null, + "advanced_permissions": null + }), + "test-workspace" + ) + .execute(db) + .await?; + Ok(()) +} + +/// Configures the primary storage through the route, which is what sweeps the workspace's +/// backups out of the instance store. +async fn configure_primary_lfs_via_route(base: &str, root_path: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": { + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": false, + "advanced_permissions": null, + "secondary_storage": {} + }})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The instance setting allowing the instance store to stand in for a workspace without +/// storage: `None` leaves it unset, which is on. +async fn set_instance_fallback(db: &Pool, on: Option) -> anyhow::Result<()> { + sqlx::query("DELETE FROM global_settings WHERE name = 'ai_sessions_instance_storage_fallback'") + .execute(db) + .await?; + if let Some(on) = on { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ('ai_sessions_instance_storage_fallback', $1)", + ) + .bind(json!(on)) + .execute(db) + .await?; + } + Ok(()) +} + +/// Polls until nothing is under the directory, for a deletion that runs off the request. +async fn wait_until_empty(dir: &std::path::Path, what: &str) { + for _ in 0..100 { + if files_under(dir).is_empty() { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("{what}: objects left under {}", dir.display()); +} + +async fn list(base: &str, token: &str) -> anyhow::Result { + let resp = authed(client().get(format!("{base}/ai/sessions/list")), token) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn pull(base: &str, token: &str, ids: &[&str]) -> anyhow::Result { + let resp = authed(client().post(format!("{base}/ai/sessions/pull")), token) + .json(&json!({ "ids": ids })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn push(base: &str, token: &str, body: Value) -> anyhow::Result { + Ok( + authed(client().post(format!("{base}/ai/sessions/push")), token) + .json(&body) + .send() + .await?, + ) +} + +fn copy_dir(from: &std::path::Path, to: &std::path::Path) -> std::io::Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.path().is_dir() { + copy_dir(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), target)?; + } + } + Ok(()) +} + +async fn rotate(base: &str, key: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/encryption_key")), + "SECRET_TOKEN", + ) + .json(&json!({ "new_key": key })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The user's prefix on disk, `windmill_ai_sessions/{w_id}/g{generation}/{email hash}`, +/// under the newest generation: deleting an older generation's objects leaves its +/// directories behind, and `read_dir` order differs across filesystems. +fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { + let workspace = storage_dir.join("windmill_ai_sessions/test-workspace"); + let hash = calculate_hash(email); + std::fs::read_dir(&workspace) + .ok() + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| { + let generation: i64 = entry + .file_name() + .to_str()? + .strip_prefix('g')? + .parse() + .ok()?; + Some((generation, entry.path().join(&hash))) + }) + .filter(|(_, path)| path.exists()) + .max_by_key(|(generation, _)| *generation) + .map(|(_, path)| path) + .expect("the user has backups under the current key") +} + +/// Every file under the storage root, as bytes. +fn files_under(root: &std::path::Path) -> Vec<(std::path::PathBuf, Vec)> { + let mut out = vec![]; + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + out.push((path.clone(), std::fs::read(&path).unwrap_or_default())); + } + } + } + out +} + +#[sqlx::test(fixtures("base"))] +async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // No storage configured, and the instance store (another test of this process may + // have loaded one) not allowed to stand in: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], false); + assert_eq!(listing["sessions"], json!([])); + + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let chat = json!({ "id": "c1", "sessionId": "s1", "title": "MARKER_PLAINTEXT_TITLE", "lastModified": 2, + "actualMessages": [], "displayMessages": [{"role": "user", "content": "hello"}] }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ + "id": "s1", + "whole": true, + "head": head, + "chats": [{ "id": "c1", "record": chat }, { "id": "c2", "record": { "id": "c2" } }], + "images": [{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }], + "artifacts": { "items": [], "versions": [] } + }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // A part more parts follow names its push, or the session would stay listed between + // the parts: one that does not is refused before anything of it lands. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "partial": true, "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s1" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .len(), + 2 + ); + + // A part with more of the session to follow lists nothing; the part that completes + // the push does, newest first. + let ids = |listing: &Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .map(|s| s["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "opens": true, "head": { "id": "s2", "workspace_id": "test-workspace", "createdAt": 2, "chatId": "c" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s2", "s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s2"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A session that outgrew one answer (three chats of 12 MB against the 32 MB budget) + // comes in pages, each naming where the next picks up, and nothing is left out. + let big = "y".repeat(12 * 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "opens": true, "head": { "id": "s4", "workspace_id": "test-workspace", "createdAt": 4, "chatId": "c1" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for cid in ["c1", "c2", "c3"] { + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "chats": [{ "id": cid, "record": { "id": cid, "big": big } }], "partial": cid != "c3" }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut pages = vec![]; + let mut resume = json!(null); + loop { + let body = if resume.is_null() { + json!({ "ids": ["s4"] }) + } else { + json!({ "ids": ["s4"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = pulled["sessions"][0].clone(); + assert_eq!(page["id"], "s4"); + resume = page["next"].clone(); + pages.push(page); + if resume.is_null() { + break; + } + assert!(pages.len() < 5, "a paged pull must end"); + } + assert!(pages.len() >= 2, "36 MB must not fit one answer"); + // Every page of an unchanged backup carries the same listing fingerprint; a chat added + // to the session changes it, which is what tells a browser its pages do not belong + // together any more. + let listing = pages[0]["listing"].clone(); + assert!(listing.is_string()); + assert!(pages.iter().all(|p| p["listing"] == listing)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 1 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + // So does a chat rewritten at the same size: the fingerprint takes in the entity tag, + // not only the size and a modification time the store may report coarsely. + let listing = pulled["sessions"][0]["listing"].clone(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 2 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + let mut chat_ids: Vec = pages + .iter() + .flat_map(|p| p["chats"].as_array().unwrap().iter()) + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect(); + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s4"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // More objects than a page keeps listing metadata for, from a store that lists in no + // order: the pages still carry every one of them, each once. + let many = 5001; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "opens": true, "head": { "id": "s5", "workspace_id": "test-workspace", "createdAt": 5, "chatId": "c00000" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for start in (0..many).step_by(100) { + let chats: Vec = (start..(start + 100).min(many)) + .map(|i| json!({ "id": format!("c{i:05}"), "record": { "id": format!("c{i:05}") } })) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "chats": chats, "partial": start + 100 < many }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut seen = std::collections::HashSet::new(); + let mut resume = json!(null); + let mut pages = 0; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s5"] }) + } else { + json!({ "ids": ["s5"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = &pulled["sessions"][0]; + for c in page["chats"].as_array().unwrap() { + assert!( + seen.insert(c["id"].as_str().unwrap().to_string()), + "a chat came twice" + ); + } + pages += 1; + resume = page["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a paged pull must end"); + } + assert_eq!(pages, 2); + assert_eq!(seen.len(), many); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s5"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A head at exactly its cap round-trips: the ciphertext read back is a block larger. + let mut big_head = json!({ "id": "s3", "workspace_id": "test-workspace", "createdAt": 3, "chatId": "c", "pad": "" }); + let pad = 1024 * 1024 - serde_json::to_string(&big_head)?.len(); + big_head["pad"] = json!("x".repeat(pad)); + assert_eq!(serde_json::to_string(&big_head)?.len(), 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s3", "whole": true, "head": big_head }] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s3"]).await?; + assert_eq!(pulled["sessions"][0]["head"], big_head); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s3"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // An object larger than any push writes, planted with the bucket's credentials at a + // predictable key, is not read. + let planted = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/planted/head.json"); + std::fs::create_dir_all(planted.parent().unwrap())?; + std::fs::File::create(&planted)?.set_len(32 * 1024 * 1024 + 1)?; + let pulled = pull(&base, "SECRET_TOKEN", &["planted"]).await?; + assert_eq!(pulled["sessions"], json!([])); + std::fs::remove_dir_all(planted.parent().unwrap())?; + // Under a session that exists, a planted chat is skipped without buffering and without + // the page ending before it, so the pull neither balloons nor loops. + let planted_chat = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s1/chats/planted.json"); + std::fs::File::create(&planted_chat)?.set_len(32 * 1024 * 1024 + 1)?; + let mut resume = json!(null); + let mut pages = 0; + let mut chat_ids = vec![]; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s1"] }) + } else { + json!({ "ids": ["s1"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + assert_eq!(pulled["sessions"][0]["head"], head); + for c in pulled["sessions"][0]["chats"].as_array().unwrap() { + chat_ids.push(c["id"].as_str().unwrap().to_string()); + } + pages += 1; + resume = pulled["sessions"][0]["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a planted object must not keep the pull going"); + } + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2"]); + std::fs::remove_file(&planted_chat)?; + + let pulled = pull(&base, "SECRET_TOKEN", &["s1", "never-pushed"]).await?; + assert_eq!(pulled["deferred"], json!([])); + let sessions = pulled["sessions"].as_array().unwrap(); + assert_eq!(sessions.len(), 1, "an id with no backup is simply absent"); + let s1 = &sessions[0]; + assert_eq!(s1["head"], head); + let mut chats = s1["chats"].as_array().unwrap().clone(); + chats.sort_by_key(|c| c["id"].as_str().unwrap().to_string()); + assert_eq!(chats[0]["record"], chat); + assert_eq!(chats[1]["id"], "c2"); + assert_eq!( + s1["images"], + json!([{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }]) + ); + assert_eq!(s1["artifacts"], json!({ "items": [], "versions": [] })); + + // Nothing on disk carries the transcript in the clear. + let files = files_under(storage_dir.path()); + assert!( + files.len() >= 4, + "expected the pushed objects on disk, got {files:?}" + ); + for (path, bytes) in &files { + let text = String::from_utf8_lossy(bytes); + assert!( + !text.contains("MARKER_PLAINTEXT_TITLE") && !text.contains("base64,AAAA"), + "{} holds plaintext", + path.display() + ); + } + let key_paths: Vec = files + .iter() + .map(|(p, _)| { + p.strip_prefix(storage_dir.path()) + .unwrap() + .to_string_lossy() + .to_string() + }) + .collect(); + assert!( + key_paths + .iter() + .all(|p| p.starts_with("windmill_ai_sessions/test-workspace/") + && !p.contains("test@windmill.dev")), + "keys carry the workspace and never the email: {key_paths:?}" + ); + + // Another member of the workspace sees none of it. + let other = list(&base, "SECRET_TOKEN_2").await?; + assert_eq!(other["enabled"], true); + assert_eq!(other["sessions"], json!([])); + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!(other["sessions"], json!([])); + + // Nor after copying the first user's ciphertext under their own prefix, which anyone + // holding the bucket credentials can do: the key is bound to the user, not the workspace. + let first = user_root(storage_dir.path(), "test@windmill.dev"); + let second = first + .parent() + .unwrap() + .join(calculate_hash("test2@windmill.dev")); + copy_dir(&first, &second)?; + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!( + other["sessions"], + json!([]), + "relocated ciphertext must not decrypt for another user" + ); + std::fs::remove_dir_all(&second)?; + + // Deleting a chat takes its images along; a head-only push leaves the rest in place. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + let s1 = &pulled["sessions"][0]; + assert_eq!(s1["head"], head); + assert_eq!(s1["chats"].as_array().unwrap().len(), 1); + assert_eq!(s1["chats"][0]["id"], "c2"); + assert_eq!(s1["images"], json!([])); + + // Rotating the workspace key moves the routes to a fresh generation's prefix and deletes + // the older ones off the request rather than re-key anything; the answers name the new + // generation (`backup_generation`), which is what makes every browser push its sessions + // whole again, while `storage_id` names the storage and stays. + let before = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + for _ in 0..100 { + if files_under(storage_dir.path()).is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + files_under(storage_dir.path()).is_empty(), + "a rotation must leave no backup object behind" + ); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert_eq!(listing["storage_id"], before["storage_id"]); + assert_ne!( + listing["backup_generation"], before["backup_generation"], + "a rotation must bump the backup generation" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"], + json!([]) + ); + // The browser's next push fills the storage back under the new key. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["sessions"][0]["head"], head); + assert_eq!(pulled["sessions"][0]["chats"][0]["id"], "c2"); + // Setting the key already in place is not a rotation the browsers would notice, so it + // keeps the backups. + let same = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["storage_id"], same["storage_id"]); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["head"], + head + ); + + // A push that does not open the session whole rides on the head in the storage; with + // none there (another device removed the backup, or nothing was ever pushed) it is + // refused and lists nothing, head or no head on it, until the session goes whole. + let s6_head = + json!({ "id": "s6", "workspace_id": "test-workspace", "createdAt": 6, "chatId": "c" }); + let not_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s6") + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "whole": true, "head": s6_head, "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!(answer["results"][0]["needs_whole"].is_null()); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s6" + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s6"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "head": s6_head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + assert!( + files_under(&user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s6")) + .is_empty() + ); + + // Between the parts of a whole push (head landed, marker not yet), an incremental push + // from another device is refused too: it rides on a listed session, and there is none + // until the last part, which lists it. + let s8_head = + json!({ "id": "s8", "workspace_id": "test-workspace", "createdAt": 8, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "opens": true, "head": s8_head, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "chats": [{ "id": "c9", "record": { "id": "c9" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s8") + }; + assert!(!listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "chats": [{ "id": "c1", "record": { "id": "c1" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s8"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect::>(), + vec!["c1"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s8"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A whole push replaces the backup: what the storage held of the session and the new + // push does not carry (a chat deleted while the workspace was on another storage) goes. + let s9_head = + json!({ "id": "s9", "workspace_id": "test-workspace", "createdAt": 9, "chatId": "c1" }); + let s9_chats = |ids: &[&str]| -> Vec { + ids.iter() + .map(|c| json!({ "id": c, "record": { "id": c } })) + .collect() + }; + let pulled_chats = |pulled: Value| -> Vec { + pulled["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "head": s9_head, "chats": s9_chats(&["c1", "c2"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c2"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "epoch": 1, "head": s9_head, "chats": s9_chats(&["c1"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1"] + ); + // The marker carries the move count the push named, once; an incremental push at + // another count rides on nothing, one at the same count lands. + let s9_epochs = |listing: Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["id"] == "s9") + .map(|s| s["epoch"].clone()) + .collect() + }; + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7"] + ); + + // An incremental push split over parts unlists the session while it is in progress (a + // pull between two parts would take a mix of old and new pieces for the backup) and + // lists it again with the last part; while one is in progress or abandoned, a push that + // is not part of it is refused, so the browser's next push of the session goes whole. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "opens": true, "chats": s9_chats(&["c8"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + Vec::::new() + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c11"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "chats": s9_chats(&["c9"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7", "c8", "c9"] + ); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + + // Two devices pushing the session whole at once: the push that opened later replaced + // the earlier one's pieces, so the earlier one's last part is refused and lists nothing, + // and the session is listed with the later push's pieces alone. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "opens": true, "head": s9_head, "chats": s9_chats(&["c3"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "opens": true, "head": s9_head, "chats": s9_chats(&["c4"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "chats": s9_chats(&["c5"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s9")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "chats": s9_chats(&["c6"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c4", "c6"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s9"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // An incremental part changing more than one object unlists the session before its + // writes, so one write failing after another landed leaves it absent rather than listed + // as a mix of old and new pieces. A directory planted at `artifacts.json` fails that write. + let s10_head = + json!({ "id": "s10", "workspace_id": "test-workspace", "createdAt": 10, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1"]), "artifacts": { "items": ["a1"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let s10_dir = user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s10"); + let artifacts_path = s10_dir.join("artifacts.json"); + std::fs::remove_file(&artifacts_path)?; + std::fs::create_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c2"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!( + answer["results"][0]["error"].is_string(), + "the artifacts write must fail: {answer}" + ); + assert!(answer["results"][0]["needs_whole"].is_null()); + assert!( + s10_dir.join("chats/c2.json").is_file(), + "the chat landed before the artifacts failed" + ); + let s10_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s10") + }; + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s10"]).await?["sessions"], + json!([]) + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c3"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + std::fs::remove_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1", "c2", "c3"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(s10_listed(list(&base, "SECRET_TOKEN").await?)); + let pulled = pull(&base, "SECRET_TOKEN", &["s10"]).await?; + assert_eq!( + pulled["sessions"][0]["artifacts"], + json!({ "items": ["a2"] }) + ); + assert_eq!(pulled_chats(pulled), vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s10"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // Removal empties both prefixes. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s1"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert!( + listing["storage_id"].is_string(), + "an answer names its storage: {listing}" + ); + assert!( + files_under(storage_dir.path()).is_empty(), + "removal must leave no object behind" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base", "jobs_read_auth"))] +async fn test_backup_writes_are_refused_for_the_wrong_owner_token_or_id( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + // A push prepared for another user must not land under the caller's prefix. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test2@windmill.dev", "sessions": [{ "id": "s1", "head": { "id": "s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 409, "{}", resp.text().await?); + + // Ids are what the server builds keys from. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "../s1", "head": { "id": "../s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A whole push opens with its head; one without is refused before anything of it lands, + // and nothing lists the session. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s7", "whole": true, "chats": [{ "id": "c", "record": { "id": "c" } }] }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s7")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["a/b"] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // An image is a base64 data URL, stored and served verbatim; anything JSON would have + // to escape (and so inflate past the pull budget) is refused. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "images": [{ "chat_id": "c1", "id": "i1", "data_url": "data:image/png;base64,\u{0001}\u{0001}\"" }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // Nested lists are bounded too: each entry is an object-store call. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": (0..1001).map(|i| format!("c{i}")).collect::>() }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // ...and across the whole request, not only per entry. + let sessions: Vec = (0..100) + .map(|i| { + json!({ "id": format!("s{i}"), "delete_chats": (0..50).map(|j| format!("c{j}")).collect::>() }) + }) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": sessions }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A pull body is a handful of ids; a large one is refused before it is parsed. + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .header("Content-Type", "application/json") + .body(format!("{{\"ids\":[\"{}\"]}}", "a".repeat(100_000))) + .send() + .await?; + assert_eq!(resp.status(), 413, "{}", resp.text().await?); + + // A scoped token (here `jobs:read`) is minted for something narrower than the user's + // whole assistant history. + let resp = authed( + client().get(format!("{base}/ai/sessions/list")), + "SCOPED_DENO_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + + assert!(files_under(storage_dir.path()).is_empty()); + Ok(()) +} + +/// Sets the object's modification time `days` back: the FilesystemStorage answers +/// `last_modified` from it, so this is a session no push touched since. +fn age_object(path: &std::path::Path, days: u64) -> std::io::Result<()> { + let at = std::time::SystemTime::now() - std::time::Duration::from_secs(days * 86_400); + std::fs::File::options() + .write(true) + .open(path)? + .set_modified(at) +} + +#[sqlx::test(fixtures("base"))] +async fn test_expired_backups_are_swept_by_age_and_left_out_of_the_listing( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + let chat = |sid: &str, cid: &str| { + json!({ "id": cid, "record": { "id": cid, "sessionId": sid, "lastModified": 2, + "actualMessages": [], "displayMessages": [] } }) + }; + let whole = |sid: &str| { + json!({ + "id": sid, "whole": true, "epoch": 0, + "head": { "id": sid, "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }, + "chats": [chat(sid, "c1")], + "images": [{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }], + "artifacts": { "items": [], "versions": [] } + }) + }; + // Two pushes split over parts of which only the first part landed: one a browser + // abandoned long ago (its token aged past the retention), one still in flight. + let opening = |sid: &str| { + json!({ + "id": sid, "whole": true, "epoch": 0, "push": format!("t-{sid}"), "opens": true, + "partial": true, "chats": [chat(sid, "c1")], + "head": { "id": sid, "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" } + }) + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [ + whole("old"), whole("live"), opening("abandoned"), opening("inflight") + ] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let root = user_root(storage_dir.path(), "test@windmill.dev"); + age_object(&root.join("index/old/0"), 40)?; + age_object(&root.join("index/abandoned/push"), 40)?; + + let listed = |listing: Value| -> Vec { + let mut ids: Vec = listing["sessions"] + .as_array() + .unwrap() + .iter() + .map(|s| s["id"].as_str().unwrap().to_string()) + .collect(); + ids.sort(); + ids + }; + let objects = |root: &std::path::Path| -> Vec { + files_under(root) + .into_iter() + .map(|(p, _)| p.strip_prefix(root).unwrap().to_string_lossy().into_owned()) + .collect() + }; + + // Without a retention nothing is swept, however old. + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live", "old"]); + + let set_retention = |days: Value| { + authed( + client().post(format!("{base}/workspaces/edit_copilot_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "sessions_retention_days": days })) + .send() + }; + let resp = set_retention(json!(0)).await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + let resp = set_retention(json!(30)).await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // The listing leaves the expired session out before the sweep reaches it. + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + assert!(root.join("sessions/old/head.json").exists()); + + // A removal cut short (a directory stands where the head is, so it cannot be unlinked) + // leaves the sweep's record with the markers gone; the next pass finds it and finishes. + let head = root.join("sessions/old/head.json"); + std::fs::remove_file(&head)?; + std::fs::create_dir(&head)?; + std::fs::write(head.join("planted"), b"")?; + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert!(root.join("index/old/sweep").exists()); + assert!(!root.join("index/old/0").exists()); + assert!(root.join("sessions/old/chats/c1.json").exists()); + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + std::fs::remove_dir_all(&head)?; + + windmill_api::sweep_expired_ai_session_backups(&db).await; + let remaining = objects(&root); + assert!( + remaining + .iter() + .all(|p| !p.contains("/old/") && !p.contains("/abandoned/")), + "{remaining:?}" + ); + for kept in [ + "index/live/0", + "sessions/live/head.json", + "sessions/live/chats/c1.json", + "images/live/c1/img1", + "index/inflight/push", + "sessions/inflight/head.json", + ] { + assert!( + remaining.iter().any(|p| p == kept), + "{kept} in {remaining:?}" + ); + } + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + + // A second pass has nothing to do; a session pushed again since its marker aged is + // renewed by the push, which rewrites the marker. + windmill_api::sweep_expired_ai_session_backups(&db).await; + age_object(&root.join("index/live/0"), 40)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", + "sessions": [{ "id": "live", "epoch": 0, "chats": [chat("live", "c2")] }] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + windmill_api::sweep_expired_ai_session_backups(&db).await; + let mut after = objects(&root); + after.sort(); + let mut expected = remaining.clone(); + expected.push("sessions/live/chats/c2.json".to_string()); + expected.sort(); + assert_eq!(after, expected); + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + Ok(()) +} + +/// Puts the process-wide instance store back to none, even when an assertion fails. +struct ResetInstanceStore; +impl Drop for ResetInstanceStore { + fn drop(&mut self) { + if let Ok(mut store) = windmill_object_store::OBJECT_STORE_SETTINGS.try_write() { + *store = None; + } + } +} + +/// Puts the process-wide license key id back to none, an Enterprise plan in this build, +/// even when an assertion fails. +struct ResetLicensePlan; +impl Drop for ResetLicensePlan { + fn drop(&mut self) { + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new(String::new())); + } +} + +/// A workspace without storage of its own backs up to the instance object store, every +/// answer saying so (`fallback`); a storage of its own, once configured, answers instead, +/// under a generation past everything the workspace left in the instance store, which the +/// change deletes; a plan switched to Pro stops the fallback with the store still loaded. +#[sqlx::test(fixtures("base"))] +async fn test_backups_fall_back_to_the_instance_storage(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // The instance store, built from its settings as `reload_object_store_setting` does. + let instance_dir = tempfile::tempdir()?; + let instance_root = instance_dir.path().to_string_lossy().to_string(); + *windmill_object_store::OBJECT_STORE_SETTINGS.write().await = Some( + windmill_object_store::build_object_store_from_settings( + windmill_object_store::ObjectSettings::Filesystem( + windmill_object_store::FilesystemSettings { root_path: instance_root.clone() }, + ), + None, + ) + .await?, + ); + let _reset = ResetInstanceStore; + let in_instance = instance_dir + .path() + .join("windmill_ai_sessions/test-workspace"); + + // Turned off by the instance setting: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + set_instance_fallback(&db, None).await?; + + // On, as it is unless turned off: the backups land in the instance store, under the + // workspace's prefix, and every answer says which kind of store it came from. + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let entry = json!({ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c1", "record": { "id": "c1" } }] }); + let push_whole = || { + push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [entry.clone()] }), + ) + }; + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["fallback"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["fallback"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + let fallback_storage_id = listing["storage_id"].clone(); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["fallback"], true); + assert_eq!(pulled["sessions"][0]["head"], head); + assert!(!files_under(&in_instance).is_empty()); + + // The workspace's storage usage counts them, under a name of their own. + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + let fallback_usage = usage["storages"] + .as_array() + .unwrap() + .iter() + .find(|s| s["storage"] == "_ai_sessions_fallback_") + .unwrap_or_else(|| panic!("no fallback usage in {usage}")); + assert!(fallback_usage["bytes"].as_i64().unwrap() > 0); + + // The retention sweep reaches what the instance store keeps for the workspace, choosing + // that store from the row it reads the generation from. + let instance_user = user_root(instance_dir.path(), "test@windmill.dev"); + age_object(&instance_user.join("index/s1/0"), 40)?; + sqlx::query( + "UPDATE workspace_settings SET ai_config = '{\"sessions_retention_days\": 30}' \ + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert!(!instance_user.join("index/s1/0").exists()); + assert!(!instance_user.join("sessions/s1/head.json").exists()); + // Pushed again, so the rotation below has a backup to delete. + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + + // A key rotation sweeps the older generation out of the instance store too. + rotate(&base, &"c".repeat(64)).await?; + wait_until_empty(&in_instance, "a rotation on the instance store").await; + assert_eq!(list(&base, "SECRET_TOKEN").await?["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + + // A storage of its own answers instead, under a generation the configuration moved past + // everything the workspace left in the instance store: nothing there is read again, + // whichever store a later return to the fallback finds, and it is deleted. + let before = list(&base, "SECRET_TOKEN").await?; + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs_via_route(&base, &storage_dir.path().to_string_lossy()).await?; + wait_until_empty(&in_instance, "configuring a workspace storage").await; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_ne!(listing["storage_id"], fallback_storage_id); + assert_eq!( + listing["backup_generation"].as_i64(), + before["backup_generation"].as_i64().map(|g| g + 1), + "configuring a storage over the fallback must move the generation on" + ); + assert_eq!(listing["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(storage_dir.path()).is_empty()); + assert!(files_under(&in_instance).is_empty()); + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage?refresh=true")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + assert!( + !usage.to_string().contains("_ai_sessions_fallback_"), + "nothing is counted in the instance store for a workspace with storage: {usage}" + ); + + // Pointed at the instance store's own bucket, a storage of its own keeps its live + // backups there under the current generation, which no storage change deletes. + configure_primary_lfs_via_route(&base, &instance_root).await?; + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + let same = list(&base, "SECRET_TOKEN").await?; + configure_primary_lfs_via_route(&base, &instance_root).await?; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + assert!(!files_under(&in_instance).is_empty()); + let listing = list(&base, "SECRET_TOKEN").await?; + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // Back to no storage of its own, the fallback answers; a plan switched to Pro while the + // instance store stays loaded stops it at once, for the listing and the push alike. + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": null })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!(list(&base, "SECRET_TOKEN").await?["fallback"], true); + let _enterprise_again = ResetLicensePlan; + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new("test_pro".to_string())); + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], false, "{pushed}"); + + Ok(()) +} diff --git a/backend/tests/ai_shared_artifacts.rs b/backend/tests/ai_shared_artifacts.rs new file mode 100644 index 0000000000..17dd313755 --- /dev/null +++ b/backend/tests/ai_shared_artifacts.rs @@ -0,0 +1,184 @@ +//! Shared AI session artifacts: one link per author and artifact, readable by any workspace +//! member until its retention window passes, and removable only by its author or an admin. +//! +//! Expiry is enforced on read as well as by the monitor's sweep, so a share past its window must +//! not be served in the gap before the sweep reaches it. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN: &str = "Bearer SECRET_TOKEN"; +const MEMBER: &str = "Bearer SECRET_TOKEN_2"; + +async fn share( + client: &reqwest::Client, + base: &str, + token: &str, + content: &str, +) -> anyhow::Result { + let resp = client + .post(format!("{base}/share")) + .header("Authorization", token) + .json(&json!({ + "artifact_id": "plan:session-1", + "name": "Plan", + "kind": "md", + "version": 1, + "content": content, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +#[sqlx::test(fixtures("base", "ai_shared_artifacts"))] +async fn shared_artifact_is_served_to_members_until_it_expires( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let first = share(&client, &base, MEMBER, "draft").await?; + let second = share(&client, &base, MEMBER, "final").await?; + assert_eq!(first["id"], second["id"], "re-sharing minted a second link"); + let id = second["id"].as_str().unwrap(); + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["content"], "final"); + + // The handlers read through the raw pool, so the workspace in the URL is the only thing + // scoping a share: a member of another workspace must not reach it by id through theirs. + let resp = client + .get(format!( + "http://localhost:{}/api/w/test-workspace-2/ai/shared_artifacts/get/{id}", + server.addr.port() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a share was served through another workspace's path" + ); + + sqlx::query( + "UPDATE ai_shared_artifact SET shared_at = now() - ($1::bigint + 60) * interval '1 second'", + ) + .bind(windmill_common::ai_shared_artifact_retention_secs()) + .execute(&db) + .await?; + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 404, "an expired share was served"); + + let status: Value = client + .get(format!("{base}/status?artifact_id=plan:session-1")) + .header("Authorization", MEMBER) + .send() + .await? + .json() + .await?; + assert!( + status.get("share").is_none(), + "an expired share was reported live: {status}" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn only_the_author_or_an_admin_can_unshare(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let shared = share(&client, &base, ADMIN, "admin's plan").await?; + let id = shared["id"].as_str().unwrap(); + + let resp = client + .delete(format!("{base}/delete/{id}")) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 404); + let remaining: i64 = sqlx::query_scalar("SELECT count(*) FROM ai_shared_artifact") + .fetch_one(&db) + .await?; + assert_eq!(remaining, 1, "a member deleted someone else's share"); + + let member_share = share(&client, &base, MEMBER, "member's plan").await?; + let resp = client + .delete(format!( + "{base}/delete/{}", + member_share["id"].as_str().unwrap() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + Ok(()) +} + +/// The id is compared against a `VARCHAR(255)` column on every route that takes one, and a +/// NUL in it would otherwise reach Postgres and come back as a 500. +#[sqlx::test(fixtures("base"))] +async fn a_malformed_artifact_id_is_refused_on_every_route( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + for bad_id in ["", "a\0b", &"x".repeat(256)] { + let resp = client + .get(format!("{base}/status")) + .query(&[("artifact_id", bad_id)]) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 400, "status accepted {bad_id:?}"); + + let resp = client + .post(format!("{base}/share")) + .header("Authorization", MEMBER) + .json(&json!({ + "artifact_id": bad_id, + "name": "Plan", + "kind": "md", + "version": 1, + "content": "x", + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "share accepted {bad_id:?}"); + } + + Ok(()) +} diff --git a/backend/tests/app_guest_cloud_hosted.rs b/backend/tests/app_guest_cloud_hosted.rs new file mode 100644 index 0000000000..7ab4145a65 --- /dev/null +++ b/backend/tests/app_guest_cloud_hosted.rs @@ -0,0 +1,168 @@ +//! Guests are unavailable on the shared cloud (`CLOUD_HOSTED`). +//! +//! One test in its own binary on purpose: `CLOUD_HOSTED` is read once into a +//! `lazy_static`, so it must be set before anything reads it and cannot be unset for a +//! sibling test in the same process. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const GUEST_TOKEN: &str = "GUEST_SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn the_cloud_admits_no_guest(db: Pool) -> anyhow::Result<()> { + // Before the server starts, so the flag is what the whole process sees. + unsafe { std::env::set_var("CLOUD_HOSTED", "true") }; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // The workspace switch cannot be turned on, so no policy can lean on it. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("self-hosted"), + "the refusal must name what guests need" + ); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "an app cannot be deployed to guests"); + + // Nor can a key be configured for the JWT way in — the refusal lands before the + // outbound JWKS fetch it would otherwise make. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({ "jwks_url": "https://issuer.example.com/.well-known/jwks.json" })) + .send() + .await?; + assert_eq!(resp.status(), 400, "a guest JWT key cannot be configured"); + // Clearing one stays allowed: a key nobody can use is still worth removing. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // An app already stored in guest mode — pushed by git-sync, or deployed before the + // instance became a cloud one — advertises no entry either. + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + sqlx::query( + "UPDATE app SET policy = jsonb_set(policy, '{execution_mode}', '\"guest\"') + WHERE path = $1 AND workspace_id = 'test-workspace'", + ) + .bind(APP_PATH) + .execute(&db) + .await?; + sqlx::query("UPDATE workspace_settings SET guest_access_enabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + + // Deploying it again is not refused: only widening an app into guests is, so a + // git-sync push of one already stored that way keeps working (and keeps being inert). + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "an app already stored in guest mode must stay deployable: {}", + resp.text().await? + ); + + let resp = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "reading the share secret must succeed"); + let secret: String = resp.text().await?; + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a guest app must not advertise entry where guests are unavailable" + ); + + // And a session issued before the instance became a cloud one stops on its next + // request: the door re-reads the switch, so the credential itself is not enough. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'GUEST_SECR', $2, 'guest@example.com', + 'guest_session', $3, 'test-workspace', now() + interval '8 hours')", + ) + .bind(GUEST_TOKEN.as_bytes()) + .bind(GUEST_TOKEN) + .bind(vec![ + "guest".to_string(), + "users:read".to_string(), + format!("apps:read:{APP_PATH}"), + format!("apps:run:{APP_PATH}"), + ]) + .execute(&db) + .await?; + // `whoami` is where an admitted guest resolves as `role: guest`, so a 401 here is + // the door refusing the credential rather than a route saying no. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest session must not authenticate where guests are unavailable" + ); + + Ok(()) +} diff --git a/backend/tests/app_run_mode_lock_strip.rs b/backend/tests/app_run_mode_lock_strip.rs new file mode 100644 index 0000000000..e2bbf631db --- /dev/null +++ b/backend/tests/app_run_mode_lock_strip.rs @@ -0,0 +1,240 @@ +//! Regression: run mode of `execute_component`'s no-id inline-`raw_code` arm runs +//! *only* the `rawscript/`-pinned `content`, dropping the caller `hash`, +//! `lock`, `modules` and `dedicated_worker` and deriving `path` server-side — +//! all of which would otherwise run or install unpinned code as the app identity. +//! Preview mode keeps honoring the caller's fields. + +use serde_json::json; +use sha2::{Digest, Sha256}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(b: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + b.header("Authorization", format!("Bearer {}", token)) +} + +const CONTENT: &str = "print('benign')\n"; +// A caller lock whose presence is the whole point: if it reaches the job, the +// worker installs it. The value only needs to be recognizable in `v2_job`. +const CALLER_LOCK: &str = "evilpkg @ file:///tmp/attacker-controlled-sdist"; +// A non-codebase-sentinel hash: if it reaches the job as `runnable_id`, the +// worker fetches (and runs) a deployed script by hash instead of the pinned +// content. It need not resolve to a real row — the guard is that it never +// becomes `runnable_id`. +const CALLER_HASH: i64 = 123456789; +// A caller path in someone else's namespace: if it reaches the job as +// `runnable_path` it redirects where the pinned content's relative imports +// resolve. Run mode must instead derive the path from `/`. +const CALLER_PATH: &str = "u/attacker/evil/comp"; + +/// The pin key `execute_component` computes for a no-id inline script: +/// `rawscript/`. +fn rawscript_pin(content: &str) -> String { + let mut h = Sha256::new(); + h.update(content); + format!("rawscript/{:x}", h.finalize()) +} + +fn inline_raw_code(hash: Option, dedicated: bool) -> serde_json::Value { + let mut rc = json!({ + "language": "python3", + "content": CONTENT, + "path": CALLER_PATH, + "lock": CALLER_LOCK, + "modules": { + "m.py": { "content": "print('x')\n", "language": "python3", "lock": CALLER_LOCK } + } + }); + if let Some(h) = hash { + rc["hash"] = json!(h); + } + if dedicated { + rc["dedicated_worker"] = json!(true); + } + rc +} + +/// Fetch `(raw_lock, args-has-_MODULES, runnable_id, tag, runnable_path)` for an +/// enqueued job. +async fn job_fields( + db: &Pool, + uuid: uuid::Uuid, +) -> anyhow::Result<(Option, bool, Option, String, Option)> { + Ok(sqlx::query_as( + "SELECT raw_lock, (args ? '_MODULES'), runnable_id, tag, runnable_path \ + FROM v2_job WHERE id = $1", + ) + .bind(uuid) + .fetch_one(db) + .await?) +} + +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_strips_caller_lock_and_modules(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip"; + let pin = format!("comp:{}", rawscript_pin(CONTENT)); + + // Deployed Viewer-mode app whose only runnable is an inline script pinned by + // content hash and with no `app_script` row — the legacy `rawscript/` + // case that reaches the no-id run-mode arm this fix touches. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { pin: { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // Run mode (no `force_viewer_static_fields`): the pin authorizes the run, but + // every caller field that selects, installs, or routes code — hash, lock, + // modules, dedicated_worker — must be dropped. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + // The args map is the other injection channel: an inline run is a + // `JobKind::Preview` job, so the worker/executors read `_MODULES` and + // `_TEMP_SCRIPT_REFS` back out of the job args. Both must be stripped. + "args": { + "_MODULES": { "m.py": { "content": "print('evil')\n", "language": "python3" } }, + "_TEMP_SCRIPT_REFS": { "../evil": "deadbeef" } + }, + "component": "comp", + "raw_code": inline_raw_code(Some(CALLER_HASH), true) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 200, + "run-mode pinned inline run must be accepted: {body}" + ); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, runnable_id, tag, runnable_path) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock, None, + "run mode must strip the caller-supplied lock" + ); + assert!( + !has_modules, + "run mode must strip caller modules (both `raw_code.modules` and an `_MODULES` arg)" + ); + let has_temp_refs: bool = + sqlx::query_scalar("SELECT (args ? '_TEMP_SCRIPT_REFS') FROM v2_job WHERE id = $1") + .bind(uuid) + .fetch_one(&db) + .await?; + assert!( + !has_temp_refs, + "run mode must strip a caller `_TEMP_SCRIPT_REFS` arg (relative-import redirect)" + ); + assert_eq!( + runnable_id, None, + "run mode must strip the caller-supplied hash (no substituting a deployed script by hash)" + ); + assert!( + !tag.starts_with("dedi:"), + "run mode must strip caller `dedicated_worker` (no routing to a path-keyed dedicated worker), got tag {tag:?}" + ); + assert_eq!( + runnable_path.as_deref(), + Some(format!("{app_path}/comp").as_str()), + "run mode must derive the path server-side, not trust the caller's (relative-import base)" + ); + + // Preview mode (editor): the caller runs their own code as themselves, so the + // lock and modules are honored — the `/jobs/run/preview`-equivalent path. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "raw_code": inline_raw_code(None, false), + "force_viewer_static_fields": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "preview must be accepted: {body}"); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, _, _, _) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock.as_deref(), + Some(CALLER_LOCK), + "preview must keep the caller-supplied lock" + ); + assert!(has_modules, "preview must keep the caller-supplied modules"); + + Ok(()) +} + +/// A bare `rawscript/` policy key (no `:` prefix, as `empty_triggerables` +/// migrates v1 policies) matches for any `component`, so run mode must not let a +/// path-traversing `component` steer the server-derived `runnable_path`. +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_rejects_traversal_component(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip_bare"; + // Bare key: no `comp:` prefix, so the pin matches regardless of `component`. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { rawscript_pin(CONTENT): { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // A component that isn't a single plain segment steers the derived path's + // base: separators and `..` traverse, and an empty one shifts it up a level. + for bad in ["../../u/attacker/evil", "..", "a/b", ""] { + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": bad, + "raw_code": inline_raw_code(None, false) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "run mode must reject component {bad:?}: got {status}: {body}" + ); + } + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index fc766253bc..63d5341ba3 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -939,6 +939,127 @@ export function main() { return midValue(); }"#, Ok(()) } +/// A run with local modules and no lock executes the bundle its lock generation built, which +/// kept the imported script's pin; the run must still load the one copy in node_modules, and +/// leave the script's own data alone even where it matches the pinned specifier. +#[sqlx::test(fixtures("base"))] +async fn test_bun_modules_run_loads_imported_pin_from_node_modules( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + insert_deployed_bun_script( + &db, + "f/pinned_import_modules/module", + 41240002, + r#"import * as isNumber from "is-number@6.0.0"; +export const ns = isNumber; +export const label = "is-number@6.0.0";"#, + ) + .await; + + let job = JobPayload::Code(RawCode { + content: r#"import * as isNumber from "is-number"; +import { ns, label } from "/f/pinned_import_modules/module"; +import { local } from "./helper"; +export function main() { return [ns === isNumber, label, local()]; }"# + .into(), + path: Some("f/pinned_import_modules/main".into()), + language: ScriptLang::Bun, + modules: Some(std::collections::HashMap::from([( + "helper.ts".to_string(), + windmill_common::scripts::ScriptModule { + content: "export const local = () => 'local';".into(), + language: ScriptLang::Bun, + lock: None, + }, + )])), + ..RawCode::default() + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + assert_eq!( + result, + serde_json::json!([true, "is-number@6.0.0", "local"]) + ); + Ok(()) +} + +async fn bun_dependency_lock(db: &Pool, port: u16, path: &str, content: &str) -> String { + let deps = RunJob::from(JobPayload::RawScriptDependencies { + script_path: path.into(), + content: content.into(), + language: ScriptLang::Bun, + }) + .run_until_complete(db, false, port) + .await + .json_result() + .unwrap(); + let Some(lock) = deps["lock"].as_str() else { + panic!("the dependency job returned no lock: {deps}"); + }; + lock.to_string() +} + +/// Bundling a locked script resolves a pinned dynamic `import()` as written. Where that fails, both +/// the dependency job and a run that finds no cached bundle must still build it, from the version +/// the lock pins; where bun tolerates the failure, the bundle must stay as written. +#[sqlx::test(fixtures("base"))] +async fn test_bun_bundles_pinned_dynamic_import(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The dependency job saves the script's bundle here; the server binary creates it at startup. + std::fs::create_dir_all(&*windmill_worker::BUN_BUNDLE_CACHE_DIR)?; + + const PATH: &str = "f/pinned_dynamic_import/main"; + // Every `script` call draws its own nonce, so each job below misses every bundle cached before + // it, this test's included, and has to build one: a cached bundle skips the build under test. + // 4.17.20 is not npm's `latest`, so a bundle that lost the pin cannot match by accident. + let script = |body: &str| { + format!( + "export async function main() {{\n {body}\n}}\n// {}", + Uuid::new_v4() + ) + }; + let import = r#"const m = await import("lodash@4.17.20"); return (m.default ?? m).VERSION;"#; + + let lock = bun_dependency_lock(&db, port, PATH, &script(import)).await; + let result = RunJob::from(JobPayload::Code(RawCode { + content: script(import), + path: Some(PATH.into()), + language: ScriptLang::Bun, + lock: Some(lock), + ..RawCode::default() + })) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + assert_eq!(result, serde_json::json!("4.17.20")); + + let tolerated = script(&format!("try {{ {import} }} catch {{ return null; }}")); + let lock = bun_dependency_lock(&db, port, PATH, &tolerated).await; + let (bundle, _) = windmill_worker::compute_bundle_local_and_remote_path( + &tolerated, + &lock, + PATH, + Some(&db), + "test-workspace", + &None, + None, + ) + .await; + assert!(std::fs::read_to_string(bundle)?.contains("lodash@4.17.20")); + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -1088,6 +1209,100 @@ export function main() { Ok(()) } +/// A deployed flow runs an inline step as the `flow_node` its deploy rewrote it into, +/// a `FlowScript` job rather than the preview job the editor runs. A workflow-as-code +/// step's `task()` children must dispatch from that kind too, as re-runs of the same +/// node, or the step passes its editor test and fails once deployed. +/// +/// The step is cached: a child that shared the parent's result-cache key would hand +/// its own result (`10`) back to the parent on resume, in place of the workflow's. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_task_dispatch_from_flow_script(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let node = FlowNodeId(3000000000000011); + let job = RunJob::from(JobPayload::FlowScript { + id: node, + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: Some(60), + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10}) + ); + + let children: Vec<(String, Option, Option)> = sqlx::query_as( + "SELECT kind::text, runnable_id, cache_ttl FROM v2_job WHERE parent_job = $1", + ) + .bind(job.id) + .fetch_all(&db) + .await?; + assert_eq!( + children, + vec![("flowscript".to_string(), Some(node.0), None)], + "the task child re-runs the parent's flow node, outside the result cache" + ); + Ok(()) +} + +/// `task(fn, { cache_ttl })` on an inline task of a deployed flow's step: the child runs +/// the parent's code with the parent's arguments, so its result-cache key carries its +/// step key, or the parent and every sibling would read its result back as their own. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_inline_task_cache_is_per_task(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let payload = || JobPayload::FlowScript { + id: FlowNodeId(3000000000000012), + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }; + + let mut children_from_cache = Vec::new(); + for _ in 0..2 { + let job = RunJob::from(payload()) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10, "tripled": 15}) + ); + let from_cache: i64 = sqlx::query_scalar( + "SELECT count(*) FROM job_logs l JOIN v2_job j ON j.id = l.job_id \ + WHERE j.parent_job = $1 AND l.logs LIKE '%found in cache%'", + ) + .bind(job.id) + .fetch_one(&db) + .await?; + children_from_cache.push(from_cache); + } + assert_eq!( + children_from_cache, + vec![0, 2], + "the second run serves each task from its own cache entry" + ); + Ok(()) +} + // ============================================================================ // Environment Variable Tests // ============================================================================ diff --git a/backend/tests/fixtures/ai_shared_artifacts.sql b/backend/tests/fixtures/ai_shared_artifacts.sql new file mode 100644 index 0000000000..5f745ed0cf --- /dev/null +++ b/backend/tests/fixtures/ai_shared_artifacts.sql @@ -0,0 +1,17 @@ +-- Layers on `base`: a second workspace the superadmin `test-user` is also a member of, so a +-- share can be requested through the wrong workspace's path by a caller the route accepts. + +INSERT INTO workspace (id, name, owner) VALUES + ('test-workspace-2', 'test-workspace-2', 'test-user'); + +INSERT INTO workspace_key(workspace_id, kind, key) VALUES + ('test-workspace-2', 'cloud', 'test-key-2'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('test-workspace-2'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace-2', 'all', 'All users', '{}'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace-2', 'test@windmill.dev', 'test-user', true, 'Admin'); diff --git a/backend/tests/fixtures/git_sync_autopull_identity.sql b/backend/tests/fixtures/git_sync_autopull_identity.sql new file mode 100644 index 0000000000..3ae7d03d0e --- /dev/null +++ b/backend/tests/fixtures/git_sync_autopull_identity.sql @@ -0,0 +1,46 @@ +-- A parent workspace whose auto-pulled repository was last saved by alice, and a fork of +-- it holding only carol, the non-admin who created it. aaron is an admin who sorts before +-- alice; bob is no longer an admin; dora is a workspace admin deactivated on the instance. +-- sam and sue are instance superadmins who are not members: sam's instance username is +-- carol's, sue's is unclaimed. + +INSERT INTO workspace (id, name, owner) VALUES ('ap-parent', 'ap-parent', 'alice@windmill.dev'); +INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ('wm-fork-feat', 'feat', 'carol@windmill.dev', 'ap-parent'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('ap-parent', 'all', 'All users', '{}'), + ('wm-fork-feat', 'all', 'All users', '{}'); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled) VALUES + ('aaron@windmill.dev', 'not-a-real-hash', 'password', false, true, 'aaron', false), + ('alice@windmill.dev', 'not-a-real-hash', 'password', false, true, 'alice', false), + ('bob@windmill.dev', 'not-a-real-hash', 'password', false, true, 'bob', false), + ('carol@windmill.dev', 'not-a-real-hash', 'password', false, true, 'carol', false), + ('dora@windmill.dev', 'not-a-real-hash', 'password', false, true, 'dora', true); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled, username) VALUES + ('sam@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sam', false, 'carol'), + ('sue@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sue', false, 'sue'); + +INSERT INTO usr (workspace_id, email, username, is_admin) VALUES + ('ap-parent', 'aaron@windmill.dev', 'aaron', true), + ('ap-parent', 'alice@windmill.dev', 'alice', true), + ('ap-parent', 'bob@windmill.dev', 'bob', false), + ('ap-parent', 'carol@windmill.dev', 'carol', false), + ('ap-parent', 'dora@windmill.dev', 'dora', true), + ('wm-fork-feat', 'carol@windmill.dev', 'carol', false); + +INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) VALUES + ('ap-parent', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'), + ('wm-fork-feat', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'); + +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('ap-parent', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false, + "auto_pull":{"enabled":true,"mode":"polling","sync_forks":true, + "enabled_by":"alice@windmill.dev"}}]}'), + ('wm-fork-feat', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false}]}'); diff --git a/backend/tests/fixtures/git_sync_autopull_recovery.sql b/backend/tests/fixtures/git_sync_autopull_recovery.sql new file mode 100644 index 0000000000..6392da50ab --- /dev/null +++ b/backend/tests/fixtures/git_sync_autopull_recovery.sql @@ -0,0 +1,10 @@ +-- A workspace whose auto-pulled repository last recorded a head-check failure while +-- already synced to head "aaa": the state a recovery write is decided on. + +INSERT INTO workspace (id, name, owner) VALUES ('ap-ws', 'ap-ws', 'test-user'); + +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('ap-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "auto_pull":{"enabled":true,"mode":"polling", + "last_synced_sha":{"main":"aaa"}, + "last_pull_status":{"success":false,"at":1,"error":"head check failed: x"}}}]}'); diff --git a/backend/tests/fixtures/git_sync_fork_credential.sql b/backend/tests/fixtures/git_sync_fork_credential.sql new file mode 100644 index 0000000000..1c1d8bd1e1 --- /dev/null +++ b/backend/tests/fixtures/git_sync_fork_credential.sql @@ -0,0 +1,55 @@ +-- A parent whose git-sync repository has a recorded credential, and the workspace +-- shapes the credential lookup and the qualification predicate have to tell apart. +-- +-- The credential itself is shared down the fork chain; the recorded *status* is +-- not, because it describes one repository and a fork can repoint its copy of the +-- resource. Forks get a status by fork creation copying it, which no fixture here +-- simulates, so a fork without one is a workspace nothing has checked yet. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('parent-ws', 'parent-ws', 'test-user', NULL), + ('fork-ws', 'fork-ws', 'test-user', 'parent-ws'), + -- A fork of a fork: the shape a fork of a dev workspace takes, and the one a + -- parent-only lookup misses. + ('deep-fork-ws', 'deep-fork-ws', 'test-user', 'fork-ws'), + ('errored-fork-ws', 'errored-fork-ws', 'test-user', 'parent-ws'), + ('orphan-ws', 'orphan-ws', 'test-user', NULL); + +-- A stored credential is encrypted with its own workspace's key. +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('parent-ws', 'cloud', 'parent-key'), + ('fork-ws', 'cloud', 'fork-key'), + ('deep-fork-ws', 'cloud', 'deep-fork-key'), + ('errored-fork-ws', 'cloud', 'errored-fork-key'), + ('orphan-ws', 'cloud', 'orphan-key'); + +-- The parent holds the credential. +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('parent-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":true,"checked_at":1788500000}}]}'), + +-- A fork inherits the repository but not the credential: this is what +-- clone_workspace_data leaves behind. + ('fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- Two levels down, so neither the credential nor its status is one hop away. + ('deep-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- A fork whose own credential has since failed. Its own standing must win over +-- the parent's healthy record rather than being papered over. + ('errored-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":false,"checked_at":1788500000, + "error":"GitLab no longer accepts this token"}}]}'), + +-- No credential and no parent to borrow one from. + ('orphan-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'); + +-- The resource each repository entry names, all pointing at the same repository. +-- The errored fork carries its token in the URL, the way a repository configured +-- by hand does: a plain remote, whatever the chain above it holds. +INSERT INTO resource (workspace_id, path, value, resource_type) VALUES + ('parent-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('deep-fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('errored-fork-ws', 'u/admin/repo', '{"url":"https://oauth2:glpat-inline@gitlab.com/grp/proj.git"}', 'git_repository'), + ('orphan-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'); diff --git a/backend/tests/fixtures/schedule_fork_conflict.sql b/backend/tests/fixtures/schedule_fork_conflict.sql new file mode 100644 index 0000000000..8e81d9ede6 --- /dev/null +++ b/backend/tests/fixtures/schedule_fork_conflict.sql @@ -0,0 +1,46 @@ +-- A three-deep fork chain whose schedule rows were cloned down at fork time. +-- The middle fork has since deleted its copy, so the leaf's schedule shares +-- its cron only with the root — the shape a direct-parent check misses. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('sfc-root', 'sfc-root', 'sfc-admin', NULL), + ('sfc-mid', 'sfc-mid', 'sfc-admin', 'sfc-root'), + ('sfc-leaf', 'sfc-leaf', 'sfc-admin', 'sfc-mid'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('sfc-root', 'cloud', 'sfc-root-key'), + ('sfc-mid', 'cloud', 'sfc-mid-key'), + ('sfc-leaf', 'cloud', 'sfc-leaf-key'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('sfc-root'), ('sfc-mid'), ('sfc-leaf'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('sfc-root', 'all', 'All users', '{}'), + ('sfc-mid', 'all', 'All users', '{}'), + ('sfc-leaf', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('sfc-admin@windmill.dev', 'x', 'password', true, true, 'SFC Admin', 'sfc-admin'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('sfc-root', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-mid', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-leaf', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'); + +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('SFC_ADMIN_TOKEN'::bytea), 'hex'), 'SFC_ADMIN_', 'SFC_ADMIN_TOKEN', 'sfc-admin@windmill.dev', 't', true); + +-- Enabling pushes the next run, which needs the scheduled script to exist. +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES + ('sfc-leaf', 'sfc-admin', 'export async function main() { return "ok" }', '{}', '', '', 'f/shared/job', 7788001, 'deno', ''); + +INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) +VALUES + ('sfc-root', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', true, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + ('sfc-leaf', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + -- A path only the leaf has: nothing above shares it. + ('sfc-leaf', 'f/shared/own', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'); + +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin; +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; diff --git a/backend/tests/fixtures/wac_flow_script.sql b/backend/tests/fixtures/wac_flow_script.sql new file mode 100644 index 0000000000..1b780c6cf4 --- /dev/null +++ b/backend/tests/fixtures/wac_flow_script.sql @@ -0,0 +1,53 @@ +-- A deployed flow whose inline bun step is workflow-as-code calling task(), in the +-- shape the deploy leaves behind: the RawScript module rewritten into a flow_node that +-- the step then runs as a FlowScript job. No lock, so the worker resolves +-- windmill-client at run time like the other bun fixtures. +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', '', '', +'f/system/wac_flow_script', +'{}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"n":{"type":"integer","description":""}},"required":[],"type":"object"}', +'{"modules":[{"id":"a","value":{"type":"flowscript","id":3000000000000011,"language":"bun","input_transforms":{"n":{"expr":"flow_input.n","type":"javascript"}}}}]}', +'system' +); + +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000011, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000011', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}); + +export const main = workflow(async (n: number) => { + const d = await double(n); + return { doubled: d }; +});' +); + +-- The same flow's step with two tasks that cache their own result. +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000012, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000012', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}, { cache_ttl: 60 }); +const triple = task(async (n: number) => { + return n * 3; +}, { cache_ttl: 60 }); + +export const main = workflow(async (n: number) => { + const d = await double(n); + const t = await triple(n); + return { doubled: d, tripled: t }; +});' +); diff --git a/backend/tests/folder_default_permissioned_as.rs b/backend/tests/folder_default_permissioned_as.rs index 7aa4f0a003..238d7cdb32 100644 --- a/backend/tests/folder_default_permissioned_as.rs +++ b/backend/tests/folder_default_permissioned_as.rs @@ -630,7 +630,7 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu ); assert_eq!( policy["on_behalf_of_email"], "group-wm_deployers@windmill.dev", - "app policy.on_behalf_of_email gets folder default email" + "the stored address is derived from that principal" ); // 7b. Admin, non-matching path — acting user diff --git a/backend/tests/git_sync_autopull_identity.rs b/backend/tests/git_sync_autopull_identity.rs new file mode 100644 index 0000000000..4e7562c3f3 --- /dev/null +++ b/backend/tests/git_sync_autopull_identity.rs @@ -0,0 +1,194 @@ +//! An automatic pull runs as the admin stamped on the repository's settings, never as +//! someone picked from the workspace, and stops once that admin is revoked. A fork's +//! pull runs as the parent's pull identity, added to the fork first. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::GitRepositorySettings; +use windmill_git_sync::{reconcile_and_enqueue_pull, reconcile_fork_branch_pull}; + +const PARENT: &str = "ap-parent"; +const FORK: &str = "wm-fork-feat"; +const REPO: &str = "$res:u/alice/repo"; + +fn repo_enabled_by(email: &str) -> GitRepositorySettings { + serde_json::from_value(serde_json::json!({ + "git_repo_resource_path": REPO, + "use_individual_branch": false, + "group_by_folder": false, + "auto_pull": { "enabled": true, "enabled_by": email } + })) + .expect("repository settings") +} + +/// `(created_by, permissioned_as, permissioned_as_email)` of every pull job in `w_id`. +async fn pull_identities( + db: &Pool, + w_id: &str, +) -> anyhow::Result)>> { + Ok(sqlx::query_as( + "SELECT created_by, permissioned_as, permissioned_as_email FROM v2_job \ + WHERE workspace_id = $1 AND kind = 'deploymentcallback'", + ) + .bind(w_id) + .fetch_all(db) + .await?) +} + +fn identity(username: &str) -> (String, String, Option) { + ( + username.to_string(), + format!("u/{username}"), + Some(format!("{username}@windmill.dev")), + ) +} + +async fn recorded_pull_error(db: &Pool, w_id: &str) -> anyhow::Result { + let git_sync: serde_json::Value = + sqlx::query_scalar("SELECT git_sync FROM workspace_settings WHERE workspace_id = $1") + .bind(w_id) + .fetch_one(db) + .await?; + Ok( + git_sync["repositories"][0]["auto_pull"]["last_pull_status"]["error"] + .as_str() + .unwrap_or_default() + .to_string(), + ) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_runs_as_the_admin_who_enabled_it(db: Pool) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("alice@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("alice")]); + Ok(()) +} + +/// bob was demoted in the workspace; dora is still a workspace admin but deactivated on +/// the instance. Neither may run the pull, and the failure lands on the status rather +/// than as an error, which a webhook delivery would turn into a failed response. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_fails_on_the_status_once_the_enabling_admin_is_revoked( + db: Pool, +) -> anyhow::Result<()> { + for email in ["bob@windmill.dev", "dora@windmill.dev"] { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by(email), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "{email} must not run the pull"); + let error = recorded_pull_error(&db, PARENT).await?; + assert!(error.contains(email), "{error}"); + } + assert!(pull_identities(&db, PARENT).await?.is_empty()); + Ok(()) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn fork_pull_runs_as_the_parent_admin_added_to_the_fork( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "the fork branch must route to the fork and enqueue" + ); + + let (is_admin, in_all): (bool, bool) = sqlx::query_as( + "SELECT u.is_admin, EXISTS (SELECT 1 FROM usr_to_group g \ + WHERE g.workspace_id = u.workspace_id AND g.usr = u.username AND g.group_ = 'all') \ + FROM usr u WHERE u.workspace_id = $1 AND u.email = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert!( + is_admin && in_all, + "alice must be an admin member of the fork" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("alice")]); + + let grants: i64 = sqlx::query_scalar( + "SELECT count(*) FROM audit_partitioned WHERE workspace_id = $1 \ + AND operation = 'users.git_sync_fork_add' AND resource = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert_eq!(grants, 1, "adding alice to the fork must be audited"); + Ok(()) +} + +/// A superadmin who is not a member runs the pull under their instance username, and +/// `u/` resolves through the workspace's members first. sam's instance username +/// is carol's, so sam's stamp must not run the pull as carol; sue's is unclaimed. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn a_non_member_superadmin_runs_the_pull_only_under_an_unclaimed_username( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sam@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "sam's username belongs to carol"); + assert!(pull_identities(&db, PARENT).await?.is_empty()); + + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sue@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("sue")]); + Ok(()) +} + +/// With no stamp on the parent, the fork pull still runs as the parent's first active +/// admin: the fork holds only its non-admin creator, so no identity resolved in the fork +/// could run it. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn unstamped_fork_pull_runs_as_the_parents_first_admin( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings SET git_sync = git_sync #- '{repositories,0,auto_pull,enabled_by}' \ + WHERE workspace_id = $1", + ) + .bind(PARENT) + .execute(&db) + .await?; + + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "an unstamped parent must still sync its forks" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("aaron")]); + Ok(()) +} diff --git a/backend/tests/git_sync_autopull_recovery.rs b/backend/tests/git_sync_autopull_recovery.rs new file mode 100644 index 0000000000..d3ba8b7c9f --- /dev/null +++ b/backend/tests/git_sync_autopull_recovery.rs @@ -0,0 +1,143 @@ +//! A recorded auto-pull failure is cleared once the tracked head is observed again +//! at the already-synced sha, and only then: the decision is taken on a snapshot, +//! so the write must re-check the stored row rather than overwrite it. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use std::collections::HashMap; +use uuid::Uuid; +use windmill_common::workspaces::AutoPullStatus; +use windmill_git_sync::{clear_auto_pull_failure, persist_auto_pull_state}; + +const WS: &str = "ap-ws"; +const REPO: &str = "$res:u/admin/repo"; + +/// The failure the fixture records, as the poller would have read it. +fn fixture_failure() -> AutoPullStatus { + AutoPullStatus { + synced_sha: None, + at: 1, + job_id: None, + success: false, + error: Some("head check failed: x".to_string()), + } +} + +fn recovered(head: &str) -> AutoPullStatus { + AutoPullStatus { + synced_sha: Some(head.to_string()), + at: 2, + job_id: None, + success: true, + error: None, + } +} + +async fn stored_auto_pull(db: &Pool) -> anyhow::Result { + let git_sync: serde_json::Value = + sqlx::query_scalar("SELECT git_sync FROM workspace_settings WHERE workspace_id = $1") + .bind(WS) + .fetch_one(db) + .await?; + Ok(git_sync["repositories"][0]["auto_pull"].clone()) +} + +/// The recovery every test below runs, decided on the fixture's failure at head +/// "aaa": live in the first test, stale in the two that move the stored state +/// first. +async fn recovery_for_the_fixture_failure(db: &Pool) -> anyhow::Result<()> { + clear_auto_pull_failure( + db, + WS, + REPO, + "main", + "aaa", + &fixture_failure(), + &recovered("aaa"), + ) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn recovery_clears_the_failure_at_the_synced_head(db: Pool) -> anyhow::Result<()> { + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_pull_status"]["success"], true); + assert!(auto_pull["last_pull_status"].get("error").is_none()); + assert_eq!(auto_pull["last_pull_status"]["synced_sha"], "aaa"); + assert_eq!( + auto_pull["last_synced_sha"]["main"], "aaa", + "the sha map is not part of a recovery write" + ); + Ok(()) +} + +/// Between the poller observing head "aaa" unchanged and its recovery write, a +/// webhook may have enqueued newer head "bbb". The stale recovery must leave that +/// optimistic state (sha, success, job id) in place; the job's completion hook +/// relies on it, and rolling the sha back would re-enqueue "bbb" on the next tick. +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn stale_recovery_leaves_a_newer_state_alone(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + let advanced = AutoPullStatus { + synced_sha: Some("bbb".to_string()), + at: 3, + job_id: Some(job_id), + success: true, + error: None, + }; + persist_auto_pull_state( + &db, + WS, + REPO, + &HashMap::from([("main".to_string(), "bbb".to_string())]), + &advanced, + ) + .await?; + + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_synced_sha"]["main"], "bbb"); + assert_eq!(auto_pull["last_pull_status"]["synced_sha"], "bbb"); + assert_eq!(auto_pull["last_pull_status"]["job_id"], job_id.to_string()); + assert_eq!(auto_pull["last_pull_status"]["at"], 3); + Ok(()) +} + +/// The head can stay at "aaa" while a newer failure is recorded (a later head +/// check, a pull job that failed). A recovery decided on the older failure must +/// not paper over the newer one, whether it differs by timestamp or, within the +/// same second, only by its error. +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn stale_recovery_keeps_a_newer_failure_at_the_same_head( + db: Pool, +) -> anyhow::Result<()> { + let same_sha = HashMap::from([("main".to_string(), "aaa".to_string())]); + for newer in [ + AutoPullStatus { + at: 5, + error: Some("head check failed: later".to_string()), + ..fixture_failure() + }, + AutoPullStatus { + error: Some("head check failed: same second".to_string()), + ..fixture_failure() + }, + ] { + persist_auto_pull_state(&db, WS, REPO, &same_sha, &newer).await?; + + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_pull_status"]["success"], false); + assert_eq!(auto_pull["last_pull_status"]["at"], newer.at); + assert_eq!( + auto_pull["last_pull_status"]["error"], + newer.error.as_deref().unwrap() + ); + } + Ok(()) +} diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs new file mode 100644 index 0000000000..9d5eb62dfb --- /dev/null +++ b/backend/tests/git_sync_fork_credential.rs @@ -0,0 +1,376 @@ +//! A fork reaches the git credential held above it in its fork chain. +//! +//! Fork creation copies the parent's git-sync repositories but not the credential, +//! which is stored per workspace so that rotation has one owner. Chains nest (a +//! fork of a dev workspace, a fork of that), so the depth-2 cases here are what +//! keep the lookup from regressing to the parent. +//! +//! The recorded *status* is not shared the same way: it describes one repository, +//! and a fork can repoint its copy of the resource, so each workspace answers from +//! its own record and gets one by fork creation copying it down. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::git_sync_ee::{ + create_repo_webhook, git_app_installations_for, git_credential_for_url, managed_pr_base_branch, + repo_provider, repo_supports_managed_git_features, set_git_credential, GitProvider, +}; +use windmill_common::workspaces::GitCredentialProvider; + +const REPO: &str = "$res:u/admin/repo"; +const URL: &str = "https://gitlab.com/grp/proj.git"; + +/// A repository is managed when a credential is held for the repository its +/// URL names now and the last check found it healthy. The recorded status is +/// keyed by resource path, so alone it would outlive a repoint; the held +/// credential alone says nothing about whether the host still accepts it. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn credential_status_is_a_workspaces_own(db: Pool) -> anyhow::Result<()> { + assert!( + !repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "a healthy status with nothing held behind it does not qualify" + ); + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert!( + repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "the workspace holding both the credential and the recorded status qualifies" + ); + assert!( + !repo_supports_managed_git_features(&db, "fork-ws", REPO).await, + "a fork borrowing the credential with no record of its own does not: the \ + status describes one repository, and this fork's resource could name another" + ); + assert!( + !repo_supports_managed_git_features(&db, "errored-fork-ws", REPO).await, + "a workspace whose own credential failed stays disqualified" + ); + Ok(()) +} + +/// The host a repository talks to is declared when its credential is stored, and +/// travels with the credential down the fork chain. +/// +/// Read from the recorded status instead, a fork answered with the default +/// provider until its own check ran, which is long enough to register a webhook +/// against the wrong receiver. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn the_provider_comes_from_the_credential_and_reaches_forks( + db: Pool, +) -> anyhow::Result<()> { + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitHub, + "with nothing stored there is no declaration to read, so the default stands" + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitLab, + "the workspace that stored it reads its own declaration" + ); + assert_eq!( + repo_provider(&db, "fork-ws", REPO).await, + GitProvider::GitLab, + "and a fork resolving that credential reads it too, without a check of its own" + ); + assert_eq!( + repo_provider(&db, "deep-fork-ws", REPO).await, + GitProvider::GitLab, + "two levels down as well" + ); + assert_eq!( + repo_provider(&db, "orphan-ws", REPO).await, + GitProvider::GitHub, + "a workspace outside the chain resolves no credential and no declaration" + ); + assert_eq!( + repo_provider(&db, "errored-fork-ws", REPO).await, + GitProvider::GitHub, + "a token written into the URL makes the repository a plain remote: the \ + parent's credential is not consulted and no host is declared" + ); + Ok(()) +} + +/// The stored credential is shared with forks and keyed by one repository. +/// +/// Both properties are the point of keeping it in `workspace_settings` under the +/// repository's identity: sharing is what stops a rotation from stranding every +/// fork on a revoked token, and the key is what stops a rewritten resource URL +/// from carrying the token to a host of the writer's choosing. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_fork_reads_an_ancestors_credential_for_the_bound_repository_only( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "the workspace that stored it reads it back" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork stores none of its own and resolves the parent's" + ); + assert_eq!( + git_credential_for_url(&db, "deep-fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork of a fork resolves the root's, two levels up" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", "https://evil.example/grp/proj.git").await?, + None, + "a resource repointed at another repository asks for that one's \ + credential and finds none" + ); + assert_eq!( + git_credential_for_url(&db, "orphan-ws", URL).await?, + None, + "a workspace with no credential and no parent resolves nothing" + ); + Ok(()) +} + +/// One repository's credential is untouched by another's. +/// +/// The key is the repository, so picking a second repository stores beside the +/// first rather than over it. Keyed by the resource instead, a workspace editing +/// one repository's resource to point somewhere else would replace the token the +/// original repository was still syncing with. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn each_repository_keeps_its_own_credential(db: Pool) -> anyhow::Result<()> { + const OTHER_URL: &str = "https://gitlab.com/grp/other.git"; + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-first", + GitCredentialProvider::Gitlab, + ) + .await?; + set_git_credential( + &db, + "parent-ws", + OTHER_URL, + "glpat-second", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-first"), + "storing a second repository's token leaves the first's in place" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second") + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-replacement", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-replacement"), + "storing the same repository again replaces rather than duplicates" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second"), + "and still leaves the other repository alone" + ); + Ok(()) +} + +/// A credential issued for `https` is not served for the `http` spelling. +/// +/// The resource holding the URL is writable by anyone with write on its path, so +/// without the scheme in the key that edit would send the token over cleartext. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_credential_is_not_served_over_a_downgraded_transport( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", "http://gitlab.com/grp/proj.git").await?, + None + ); + Ok(()) +} + +/// A GitLab the server cannot reach is the error reported, not the GitHub App +/// lookup that runs after it: for a self-managed GitLab behind a firewall or an +/// untrusted certificate, "no GitHub App installation" names neither the host +/// nor the cause. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn an_unreachable_gitlab_host_is_the_reported_error( + db: Pool, +) -> anyhow::Result<()> { + let err = create_repo_webhook( + &db, + "parent-ws", + "http://glpat-secret@127.0.0.1:1/grp/proj.git", + "https://windmill.example/api/w/parent-ws/git_sync/webhook/gitlab", + "hook-secret", + ) + .await + .expect_err("nothing listens on port 1"); + assert!( + err.to_string().contains("Could not reach the git host"), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("glpat-secret"), + "the URL credential leaked into the error: {err}" + ); + Ok(()) +} + +/// GitHub App installations are normally copied into a fork, but a workspace +/// attached as a dev workspace, or forked before its parent connected the App, +/// holds none, and neither does anything forked from it. The lookup reaches the +/// nearest workspace up the chain that holds some, and the background App path +/// (PR base resolution here) authenticates with that installation's token. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn app_installations_come_from_the_nearest_ancestor_holding_some( + db: Pool, +) -> anyhow::Result<()> { + use axum::{routing::get, Router}; + use std::sync::{Arc, Mutex}; + + // A stand-in GitHub API: one repository, and a record of who asked for it. + let seen: Arc>> = Arc::new(Mutex::new(vec![])); + let app = Router::new().route( + "/api/v3/repos/acme/repo", + get({ + let seen = seen.clone(); + move |headers: axum::http::HeaderMap| { + let seen = seen.clone(); + async move { + let auth = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + seen.lock().unwrap().push(auth); + axum::Json(serde_json::json!({ "default_branch": "trunk" })) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let stub = format!("http://127.0.0.1:{port}"); + + // The root holds the installation, with a cached token so nothing is minted. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'parent-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 42, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "root-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + // The fork's copy of the resource names the App-backed repository. + sqlx::query("UPDATE resource SET value = $1::jsonb WHERE workspace_id = 'deep-fork-ws' AND path = 'u/admin/repo'") + .bind(serde_json::json!({ "url": format!("{stub}/acme/repo.git"), "is_github_app": true })) + .execute(&db) + .await?; + + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?, + ("parent-ws".to_string(), vec![(42, Some(stub.clone()))]), + "two levels down, the root's installations are the ones to use" + ); + assert_eq!( + git_app_installations_for(&db, "orphan-ws").await?, + ("orphan-ws".to_string(), vec![]), + "a workspace with nothing above it resolves nothing" + ); + assert_eq!( + managed_pr_base_branch(&db, "deep-fork-ws", REPO) + .await? + .as_deref(), + Some("trunk"), + "the background App path reaches the repository through the root's installation" + ); + let seen = seen.lock().unwrap().clone(); + assert!( + !seen.is_empty() && seen.iter().all(|auth| auth == "Bearer root-token"), + "every call authenticated with the root's cached token: {seen:?}" + ); + + // A closer holder takes precedence over the root. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'fork-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 7, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "mid-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?.0, + "fork-ws", + "the nearest holder wins over the root" + ); + Ok(()) +} diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index ddd94ea079..64d7854173 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -1442,7 +1442,10 @@ async fn declarative_sync_rejects_an_unusable_webhook_base_url(db: Pool) { + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + let mut desired = BTreeMap::new(); + desired.insert( + "base_url".to_string(), + serde_json::json!("https://wm.example.com"), + ); + desired.insert( + "instance_banner".to_string(), + serde_json::json!({ "enabled": true, "message": "down", "link": "javascript:alert(1)" }), + ); + + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err("a javascript: banner link must fail the sync"); + assert!( + err.to_string().contains("instance_banner"), + "the error should name the offending setting, got: {err}" + ); + + assert_eq!( + count_global_settings(&db).await, + before, + "validation must run before anything is applied" + ); + assert!( + get_global_setting(&db, "base_url").await.is_none(), + "the other settings in the same apply must not have been written either" + ); +} + +#[sqlx::test(fixtures("base"))] +async fn declarative_sync_rejects_an_unusable_default_allowed_origins(db: Pool) { + // The declarative writers (the sync-config CLI, the operator's ConfigMap + // sync) do not run the HTTP layer's pre-write hook, so an origin list that + // cannot be parsed would persist here, be dropped at boot, and leave the + // instance with no restriction at all. + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + for bad in [ + serde_json::json!([""]), + serde_json::json!(["https://a.example,https://b.example"]), + serde_json::json!("null"), + ] { + let mut desired = BTreeMap::new(); + desired.insert( + "http_route_default_allowed_origins".to_string(), + bad.clone(), + ); + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err(&format!("{bad} must fail the sync")); + assert!( + err.to_string() + .contains("http_route_default_allowed_origins"), + "the error should name the offending setting, got: {err}" + ); + } + + assert_eq!( + count_global_settings(&db).await, + before, + "a rejected sync must not have persisted anything" + ); + + // A usable list still syncs. + let mut desired = BTreeMap::new(); + desired.insert( + "http_route_default_allowed_origins".to_string(), + serde_json::json!(["https://app.example.com"]), + ); + windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect("a valid origin list must sync"); +} diff --git a/backend/tests/jobs_cross_site_get.rs b/backend/tests/jobs_cross_site_get.rs new file mode 100644 index 0000000000..20aa09a0a5 --- /dev/null +++ b/backend/tests/jobs_cross_site_get.rs @@ -0,0 +1,138 @@ +//! Regression test for cross-site GET CSRF on the job-run endpoints that can run a Hub script. +//! +//! `run_wait_result/p/{path}` and `run_and_stream/p/{path}` answer GET and, for a `hub/` path, +//! run any public Hub script. The session cookie is `SameSite=Lax`, so a browser attaches it +//! to a cross-site top-level GET navigation, and an argument written `$var:` or +//! `$res:` is resolved as the caller: an attacker page could make a logged-in browser +//! run a generic Hub script and hand it the victim's secrets. CORS hides the response but not +//! the side effect. +//! +//! `CrossSiteGetGuard` refuses such a request. Workspace scripts are deliberately not refused, +//! since they only run code the workspace's own members deployed. A request carrying its own +//! credential is allowed; the one case that regresses is a signed-in user clicking a `?token=` +//! Hub-script link from another site, because `extract_token` gives the cookie precedence and +//! exempting the parameter would let `?token=junk` reinstate the vector. +//! +//! This test pins down: +//! - both endpoints refuse a cross-site cookie GET to a Hub script (the core fix), whether +//! `Sec-Fetch-Site` says so or, with no such header (plain http), a cross-host `Referer`, +//! - a junk `token` query parameter does not buy a pass, +//! - the scope: the same request to a workspace script is not refused, +//! - a Hub-script request with its own credential (bearer, or `?token=` and no cookie), or +//! sent as a POST, gets through. +//! +//! No runnable exists and no Hub is contacted. A request that gets past the guard fails as +//! not-found on a workspace path, and on the non-numeric version in `hub/x/...` for a Hub +//! path, which is rejected while resolving the runnable, before any call to the Hub. + +use reqwest::StatusCode; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const HUB_GETS: [&str; 2] = [ + "run_wait_result/p/hub/x/absent", + "run_and_stream/p/hub/x/absent", +]; +const WORKSPACE_GETS: [&str; 2] = [ + "run_wait_result/p/u/test-user/absent", + "run_and_stream/p/u/test-user/absent", +]; + +async fn send(req: reqwest::RequestBuilder) -> anyhow::Result<(StatusCode, String)> { + let resp = req.send().await?; + let status = resp.status(); + Ok((status, resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn test_cross_site_get_cannot_run_hub_scripts(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/jobs", + server.addr.port() + ); + let client = reqwest::Client::new(); + let cookie_get = |path: &str| { + client + .get(format!("{base}/{path}")) + .header("Cookie", "token=SECRET_TOKEN") + }; + + // ---- CORE REGRESSION: a cross-site cookie GET cannot run a Hub script. + for path in HUB_GETS { + let refused = [ + ( + "Sec-Fetch-Site: cross-site", + cookie_get(path).header("Sec-Fetch-Site", "cross-site"), + ), + // Plain http gets no `Sec-Fetch-*` at all, so `Referer` is the only signal left. + ( + "cross-host Referer with no Sec-Fetch-Site", + cookie_get(path).header("Referer", "http://attacker.example/page"), + ), + // The cookie outranks a `token` query parameter when authenticating. + ( + "junk ?token= next to the cookie", + client + .get(format!("{base}/{path}?token=junk")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ]; + for (name, req) in refused { + let (status, body) = send(req).await?; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "{path} [{name}] must be refused: {body}" + ); + } + } + + // ---- Scope: the same request to a workspace script is not refused. + for path in WORKSPACE_GETS { + let (status, body) = send(cookie_get(path).header("Sec-Fetch-Site", "cross-site")).await?; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{path} is a workspace script and must reach the handler: {body}" + ); + } + + // ---- A Hub-script request that carries its own credential, or is a POST, gets through. + let hub = HUB_GETS[1]; + let allowed = [ + ( + "cross-origin bearer token", + client + .get(format!("{base}/{hub}")) + .header("Authorization", "Bearer SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "cross-origin ?token= with no cookie", + client + .get(format!("{base}/{hub}?token=SECRET_TOKEN")) + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "POST with the cookie", + client + .post(format!("{base}/{hub}")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site") + .json(&serde_json::json!({})), + ), + ]; + for (name, req) in allowed { + let (status, body) = send(req).await?; + assert!( + body.contains("Invalid hub script version"), + "{name} must get past the guard to runnable resolution (got {status}): {body}" + ); + } + + Ok(()) +} diff --git a/backend/tests/nativets_dedicated.rs b/backend/tests/nativets_dedicated.rs index 1c6987a480..1e9aef3228 100644 --- a/backend/tests/nativets_dedicated.rs +++ b/backend/tests/nativets_dedicated.rs @@ -12,7 +12,7 @@ mod prewarmed_isolate_tests { use windmill_worker::{build_loader, LoaderMode, BUN_PATH}; fn default_annotation() -> NativeAnnotation { - NativeAnnotation { useragent: None, proxy: None } + NativeAnnotation::default() } /// Bundle a TypeScript script into JS suitable for `PrewarmedIsolate`. diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index 3a81f31021..1a4040d390 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -627,3 +627,121 @@ async fn test_root_job_span_relocated_to_inbound_trace() { expected_uuid_trace ); } + +// ═══════════════════════════════════════════════════════════════════════ +// RESOURCE ATTRIBUTES (OTEL_RESOURCE_ATTRIBUTES) +// ═══════════════════════════════════════════════════════════════════════ + +fn resource_attrs() -> std::collections::HashMap { + otlp_service_resource( + &windmill_common::utils::Mode::Worker, + "fallback-host", + "dev", + ) + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_merges_env_attributes_without_losing_windmill_identity() { + // These take precedence over the hostname argument and over OTEL_RESOURCE_ATTRIBUTES, + // so clear them or an ambient one fails the assertions below for an unrelated reason. + for var in [ + "OTEL_HOST_NAME", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + ] { + std::env::remove_var(var); + } + std::env::set_var( + "OTEL_RESOURCE_ATTRIBUTES", + "k8s.pod.uid=abc-123,service.name=injected,host.name=injected", + ); + let attrs = resource_attrs(); + std::env::remove_var("OTEL_RESOURCE_ATTRIBUTES"); + + // Attributes the deployment injects reach the exporters. + assert_eq!( + attrs.get("k8s.pod.uid").map(String::as_str), + Some("abc-123") + ); + // OTEL_RESOURCE_ATTRIBUTES is the secondary resource, so Windmill's own values still win. + assert_eq!( + attrs.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + attrs.get("host.name").map(String::as_str), + Some("fallback-host") + ); +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_dedicated_overrides_win() { + // A deployment sets these per pod, e.g. from Kubernetes downward-API labels. The + // competing service.name must lose: the spec ranks OTEL_SERVICE_NAME above it. + std::env::set_var("OTEL_RESOURCE_ATTRIBUTES", "service.name=should-lose"); + std::env::set_var("OTEL_SERVICE_NAME", "windmill-workers"); + std::env::set_var("OTEL_SERVICE_VERSION", "1.802.0"); + std::env::set_var("OTEL_HOST_NAME", "pod-7"); + let overridden = resource_attrs(); + + // An empty value means unset, which is what the downward API yields for a missing label. + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + ] { + std::env::set_var(var, ""); + } + let empty = resource_attrs(); + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_RESOURCE_ATTRIBUTES", + ] { + std::env::remove_var(var); + } + let unset = resource_attrs(); + + assert_eq!( + overridden.get("service.name").map(String::as_str), + Some("windmill-workers") + ); + assert_eq!( + overridden.get("service.version").map(String::as_str), + Some("1.802.0") + ); + assert_eq!( + overridden.get("host.name").map(String::as_str), + Some("pod-7") + ); + + assert_eq!( + empty.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + empty.get("host.name").map(String::as_str), + Some("fallback-host") + ); + assert_eq!( + empty.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); + + // With nothing set at all — the default deployment — SdkProvidedResourceDetector still + // contributes service.name = "unknown_service". Ours has to overwrite it. + assert_eq!( + unset.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + unset.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); +} diff --git a/backend/tests/preserve_on_behalf_of.rs b/backend/tests/preserve_on_behalf_of.rs index 2a1f3895ca..0c8c0b76a3 100644 --- a/backend/tests/preserve_on_behalf_of.rs +++ b/backend/tests/preserve_on_behalf_of.rs @@ -405,11 +405,12 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { // 7. App: Admin preserves on_behalf_of // ======================================== + // Principal only, so the stored address can only have come from deriving it. let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") .json(&new_app_with_on_behalf_of( "u/test-user/app_admin_preserve", Some("u/original-user"), - Some("original@windmill.dev"), + None, true, )) .send() @@ -434,10 +435,24 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/original-user"), "Admin should preserve app on_behalf_of" ); + // The address is written through from the principal, never taken from the request, so the + // stored copy can only agree with it. assert_eq!( policy.get("on_behalf_of_email").and_then(|v| v.as_str()), Some("original@windmill.dev"), - "Admin should preserve app on_behalf_of_email" + "the stored address is derived from the principal, not the one the client sent" + ); + let resp = authed( + client().get(format!("{base}/apps/get/p/u/test-user/app_admin_preserve")), + "SECRET_TOKEN", + ) + .send() + .await?; + let returned: serde_json::Value = resp.json().await?; + assert_eq!( + returned["policy"]["on_behalf_of_email"].as_str(), + Some("original@windmill.dev"), + "the response returns the address written through from the principal" ); // ======================================== @@ -476,11 +491,6 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/original-user"), "Deployer should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Deployer should preserve app on_behalf_of_email" - ); // ======================================== // 9. App: Non-admin cannot preserve @@ -518,10 +528,26 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/test-user-2"), "Non-admin should have their own permissioned_as as app on_behalf_of" ); + + // ======================================== + // 9b. App: a policy naming two different accounts is rejected + // ======================================== + + // This is the shape a workspace deploy produces when it carries the source + // workspace's principal beside the target's address. + let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") + .json(&new_app_with_on_behalf_of( + "u/test-user/app_mismatched_pair", + Some("u/original-user"), + Some("test2@windmill.dev"), + true, + )) + .send() + .await?; assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("test2@windmill.dev"), - "Non-admin should have their own email as app on_behalf_of_email" + resp.status(), + 400, + "a policy whose two halves name different accounts must be rejected" ); // ======================================== @@ -1238,11 +1264,6 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/original-user"), "Admin update should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Admin update should preserve app on_behalf_of_email" - ); // ======================================== // Deployer updates with preserve flag @@ -1304,11 +1325,6 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/original-user"), "Deployer update should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Deployer update should preserve app on_behalf_of_email" - ); // ======================================== // Non-admin cannot preserve on update @@ -1370,10 +1386,62 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/test-user-2"), "Non-admin update should overwrite app on_behalf_of with their own" ); + + Ok(()) +} + +/// A superadmin acting outside their workspaces has no `usr` row, so the per-workspace rename +/// sweep never reaches the apps that name them. Their principal is their instance username, so +/// without a global sweep a rename leaves those apps naming an account that resolves to nobody. +#[sqlx::test(fixtures("preserve_on_behalf_of"))] +async fn test_rename_sweeps_external_superadmin_app_identity( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + let path = "u/original-user/app_run_by_external_superadmin"; + + let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") + .json(&new_app_with_on_behalf_of( + path, + Some("u/superadmin-external"), + Some("superadmin-external@windmill.dev"), + true, + )) + .send() + .await?; assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("test2@windmill.dev"), - "Non-admin update should overwrite app on_behalf_of_email with their own" + resp.status(), + 201, + "Should create app: {}", + resp.text().await? + ); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/users/rename/superadmin-external@windmill.dev" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "new_username": "superadmin_renamed" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "Should rename: {}", resp.text().await?); + + let app = sqlx::query!( + "SELECT policy FROM app WHERE path = $1 AND workspace_id = $2", + path, + "test-workspace" + ) + .fetch_one(&db) + .await?; + assert_eq!( + app.policy.get("on_behalf_of").and_then(|v| v.as_str()), + Some("u/superadmin_renamed"), + "the rename should follow the principal an app names" ); Ok(()) @@ -2855,10 +2923,8 @@ async fn test_reject_reserved_sentinel_on_behalf_of(db: Pool) -> anyho resp.text().await? ); - // App: a real superadmin on_behalf_of is *allowed* at deploy (deployers may - // deploy on behalf of any real user). The escalation is closed at execution - // by the job-token cap, not by restricting what can be stored, so even a - // superadmin email pinned onto an unrelated principal deploys fine here. + // App: a real superadmin's address pinned onto an unrelated principal is a pair naming two + // accounts, which the principal-authoritative policy refuses. let resp = authed( client().post(format!("{base}/apps/create")), "DEPLOYER_TOKEN", @@ -2873,12 +2939,14 @@ async fn test_reject_reserved_sentinel_on_behalf_of(db: Pool) -> anyho .await?; assert_eq!( resp.status(), - 201, - "a real superadmin on_behalf_of is allowed at deploy (capped at execution): {}", + 400, + "a superadmin address beside an unrelated principal must be refused: {}", resp.text().await? ); - // App: a consistently named real superadmin identity is likewise allowed. + // App: a real superadmin on_behalf_of is allowed at deploy when consistently named (deployers + // may deploy on behalf of any real user); the escalation is closed at execution by the + // job-token cap, not by restricting what can be stored. let resp = authed( client().post(format!("{base}/apps/create")), "DEPLOYER_TOKEN", diff --git a/backend/tests/schedule_fork_conflict.rs b/backend/tests/schedule_fork_conflict.rs new file mode 100644 index 0000000000..9a7ae8ff6d --- /dev/null +++ b/backend/tests/schedule_fork_conflict.rs @@ -0,0 +1,59 @@ +//! Enabling a schedule in a fork warns about every ancestor sharing the path, +//! not only the direct parent: the row was cloned down the whole chain, so the +//! cron is shared with whichever ancestors still hold a copy. + +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn set_enabled( + base: &str, + path: &str, + enabled: bool, + force: bool, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(format!("{base}/api/w/sfc-leaf/schedules/setenabled/{path}")) + .header("Authorization", "Bearer SFC_ADMIN_TOKEN") + .json(&json!({ "enabled": enabled, "force": force })) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("schedule_fork_conflict"))] +async fn enabling_in_a_fork_names_the_nearest_ancestor_sharing_the_path( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}", server.addr.port()); + + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-root"), + "the middle fork deleted its copy, so the root is the one still sharing the cron: {body}" + ); + + let (status, body) = set_enabled(&base, "f/shared/own", true, false).await?; + assert_eq!( + status, 200, + "a path nothing upstream has enables freely: {body}" + ); + + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) + VALUES ('sfc-mid', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin')", + ) + .execute(&db) + .await?; + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-mid"), + "with the parent holding a copy again, it is the nearest and gets named: {body}" + ); + Ok(()) +} diff --git a/backend/tests/script_auto_parent_archived.rs b/backend/tests/script_auto_parent_archived.rs index 5115c516c2..58b926b9c4 100644 --- a/backend/tests/script_auto_parent_archived.rs +++ b/backend/tests/script_auto_parent_archived.rs @@ -9,8 +9,9 @@ //! `lineage must be linear: no 2 scripts can have the same parent` error //! whenever that archived hash already had a child from the prior rename. //! -//! The fix clears `parent_hash` to `None` in that case so the push starts a -//! fresh lineage instead of failing. +//! Resolution only ever adopts an archived version nothing else descends from, +//! so here it finds none and leaves `parent_hash` at `None`, starting a fresh +//! lineage instead of failing. use serde_json::json; use sqlx::{Pool, Postgres}; @@ -146,3 +147,221 @@ async fn test_auto_parent_starts_fresh_lineage_when_all_versions_archived( Ok(()) } + +/// A path whose only versions are archived and childless — what archiving a path +/// leaves behind, and what a sync push that applies a deletion before the matching +/// update sees. Resolving `auto_parent` to no parent there hashes the deploy exactly +/// as the path's first version was hashed, so an unchanged push is rejected as a +/// duplicate of it. +#[sqlx::test(fixtures("base"))] +async fn test_auto_parent_adopts_archived_head_instead_of_colliding( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_archived_head"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let first_hash: i64 = + sqlx::query_scalar("SELECT hash FROM script WHERE path = $1 AND workspace_id = $2") + .bind(path) + .bind("test-workspace") + .fetch_one(&db) + .await?; + + let resp = authed( + client().post(format!("{base}/scripts/archive/p/{path}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "archiving the path should succeed"); + + // Byte-identical to the first deploy: the collision this guards against needs the + // pushed body to hash the same way the original one did. + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&{ + let mut push = body.clone(); + push["auto_parent"] = json!(true); + push + }) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "re-pushing an archived path must not collide with its own first version, \ + got {status}: {response_body}" + ); + + let active: Vec>> = sqlx::query_scalar( + "SELECT parent_hashes FROM script \ + WHERE path = $1 AND archived = false AND workspace_id = $2", + ) + .bind(path) + .bind("test-workspace") + .fetch_all(&db) + .await?; + assert_eq!(active.len(), 1, "exactly one active version expected"); + assert_eq!( + active[0].as_deref(), + Some(&[first_hash][..]), + "the revived version should continue the archived lineage" + ); + + Ok(()) +} + +/// The same redeploy from a caller that names no parent at all — the shape a retried +/// `wmill sync push` takes, once the archive it applied has committed and the path has +/// dropped out of the listing it diffs against. +/// +/// The adopted version supplies the lineage and nothing else: a path is reusable by a +/// different script, which must not start life holding grants nobody gave it. +#[sqlx::test(fixtures("base"))] +async fn test_parentless_redeploy_adopts_the_lineage_but_not_the_grants( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_retired_parentless"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let first_hash: i64 = + sqlx::query_scalar("SELECT hash FROM script WHERE path = $1 AND workspace_id = $2") + .bind(path) + .bind("test-workspace") + .fetch_one(&db) + .await?; + + sqlx::query("UPDATE script SET extra_perms = $1 WHERE hash = $2 AND workspace_id = $3") + .bind(json!({ "u/someone_else": true })) + .bind(first_hash) + .bind("test-workspace") + .execute(&db) + .await?; + + let resp = authed( + client().post(format!("{base}/scripts/archive/p/{path}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "archiving the path should succeed"); + + // Byte-identical to the first deploy and naming no parent: hashed as a first deploy it + // lands on the archived row. + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "a parentless redeploy of a retired path must not collide with its own first \ + version, got {status}: {response_body}" + ); + + let live: Vec<(Option>, serde_json::Value)> = sqlx::query_as( + "SELECT parent_hashes, extra_perms FROM script \ + WHERE path = $1 AND archived = false AND workspace_id = $2", + ) + .bind(path) + .bind("test-workspace") + .fetch_all(&db) + .await?; + assert_eq!(live.len(), 1, "exactly one live version expected"); + assert_eq!( + live[0].0.as_deref(), + Some(&[first_hash][..]), + "the redeploy should continue the retired lineage" + ); + assert_eq!( + live[0].1, + json!({}), + "an adopted version's grants must not carry over to whatever reuses its path" + ); + + Ok(()) +} + +/// A soft delete keeps the row, and with it the hash the version was deployed under, so a +/// tombstone has to stay adoptable: skip it and the redeploy hashes straight back onto it. +#[sqlx::test(fixtures("base"))] +async fn test_redeploy_chains_past_a_deleted_version(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_deleted_version"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let deleted_hash = resp.text().await?; + + let resp = authed( + client().post(format!("{base}/scripts/delete/h/{deleted_hash}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "deleting the version should succeed"); + + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "redeploying the content of a deleted version must not collide with its \ + tombstone, got {status}: {response_body}" + ); + + Ok(()) +} diff --git a/backend/tests/script_modules.rs b/backend/tests/script_modules.rs index 8755ebfb57..eec9d3b45a 100644 --- a/backend/tests/script_modules.rs +++ b/backend/tests/script_modules.rs @@ -159,3 +159,58 @@ export function main(name: string) { assert_eq!(result, json!("hello world")); Ok(()) } + +/// A multi-file script run without a lock is bundled by the lockfile build. A pinned import in +/// a workspace script it imports must be installed at that version and still resolve at run time. +#[sqlx::test(fixtures("base"))] +async fn test_bun_module_imports_pinned_workspace_script(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{}', '', '', 'f/system/pinned_module', 12350, 'bun', '')", + ) + .bind( + r#" +import _ from "lodash@4.17.20"; +export function lodashVersion() { return _.VERSION; } +"#, + ) + .execute(&db) + .await?; + + let mut modules = HashMap::new(); + modules.insert( + "helper.ts".to_string(), + ScriptModule { + content: "export function label(v: string) { return v; }\n".to_string(), + language: ScriptLang::Bun, + lock: None, + }, + ); + + let job = JobPayload::Code(RawCode { + content: r#" +import { lodashVersion } from "/f/system/pinned_module"; +import { label } from "./helper.ts"; +export function main() { return label(lodashVersion()); } +"# + .to_owned(), + path: Some("f/system/my_script".to_string()), + language: ScriptLang::Bun, + modules: Some(modules), + tag: None, + ..RawCode::default() + }); + + let result = RunJob::from(job) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!("4.17.20")); + Ok(()) +} diff --git a/backend/tests/session_workspace_status.rs b/backend/tests/session_workspace_status.rs index b4fc98da5e..2ed4ac503a 100644 --- a/backend/tests/session_workspace_status.rs +++ b/backend/tests/session_workspace_status.rs @@ -3,18 +3,23 @@ //! extractor actually grants. Membership is not the only path: a superadmin is authed into //! any existing workspace without a `usr` row, and `admins` has no `usr` rows at all, so //! answering from `usr` alone reports live workspaces as unresolvable and the client deletes -//! sessions that still work. +//! sessions that still work. `POST /workspaces/session_workspace_retention`, the AI session +//! retention the same client deletes its own copies by, is a workspace setting and answers to +//! the stricter bar, which is why the two are separate routes and tested together. use serde_json::json; use sqlx::{Pool, Postgres}; use std::collections::HashMap; use windmill_test_utils::*; -async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { +async fn post( + port: u16, + route: &str, + token: &str, + ids: &[&str], +) -> anyhow::Result { let resp = reqwest::Client::new() - .post(format!( - "http://localhost:{port}/api/workspaces/session_workspace_status" - )) + .post(format!("http://localhost:{port}/api/workspaces/{route}")) .header("Authorization", format!("Bearer {token}")) .json(&json!({ "workspace_ids": ids })) .send() @@ -23,6 +28,14 @@ async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result anyhow::Result> { + post(port, "session_workspace_status", token, ids).await +} + +async fn retention(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { + post(port, "session_workspace_retention", token, ids).await +} + #[sqlx::test(fixtures("base", "session_workspace_status"))] async fn test_superadmin_reaches_workspaces_without_a_usr_row( db: Pool, @@ -60,3 +73,48 @@ async fn test_superadmin_reaches_workspaces_without_a_usr_row( Ok(()) } + +/// The retention a browser deletes its own copies by is a workspace setting, so unlike the +/// status it is told only to a caller the authed extractor would let in. +#[sqlx::test(fixtures("base", "session_workspace_status"))] +async fn test_session_retention_is_told_only_to_members_who_can_be_authed( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let ids = ["foreign-workspace", "test-workspace", "no-such-workspace"]; + sqlx::query( + "UPDATE workspace_settings SET ai_config = '{\"sessions_retention_days\": 7}' \ + WHERE workspace_id IN ('test-workspace', 'foreign-workspace')", + ) + .execute(&db) + .await?; + + // test@windmill.dev is a superadmin: authed into every workspace that exists. + let sa = retention(port, "SECRET_TOKEN", &ids).await?; + assert_eq!(sa["test-workspace"], 7); + assert_eq!(sa["foreign-workspace"], 7); + assert!(!sa.contains_key("no-such-workspace")); + + // test2@windmill.dev is a member of test-workspace only. + let usr = retention(port, "SECRET_TOKEN_2", &ids).await?; + assert_eq!(usr["test-workspace"], 7); + assert!(!usr.contains_key("foreign-workspace")); + + // A disabled membership still reconciles its sessions — the status stays `active` — but + // cannot be authed into the workspace, so it is told no setting. + sqlx::query("UPDATE usr SET disabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + assert_eq!( + status(port, "SECRET_TOKEN_2", &ids).await?["test-workspace"], + "active" + ); + assert!(!retention(port, "SECRET_TOKEN_2", &ids) + .await? + .contains_key("test-workspace")); + + Ok(()) +} diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs index 8ecf6e98b6..e93979e9c7 100644 --- a/backend/tests/volume_tests.rs +++ b/backend/tests/volume_tests.rs @@ -537,6 +537,64 @@ fn test_asset_kind_volume_variant() { #[cfg(feature = "parquet")] #[sqlx::test(fixtures("base"))] async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + let code = r#"// volume: test-vol data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("data/hello.txt", "utf-8"); + writeFileSync("data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("data/output.txt"), + }; +}"#; + run_volume_with_default_stack(db, ScriptLang::Bun, code).await +} + +#[cfg(all(feature = "parquet", feature = "private", feature = "php"))] +#[sqlx::test(fixtures("base"))] +async fn test_php_volume_with_default_stack(db: Pool) -> anyhow::Result<()> { + // Nested calls exercise the parser's fallback chain; flattening them weakens this guard. + let code = r#" $content, + "output_exists" => in_array("output.txt", array_values(array_diff(scandir("data"), ['.', '..']))), + ]; +}"#; + + run_volume_with_default_stack(db, ScriptLang::Php, code).await +} + +#[cfg(feature = "parquet")] +async fn run_volume_with_default_stack( + db: Pool, + language: ScriptLang, + code: &'static str, +) -> anyhow::Result<()> { + // CI raises RUST_MIN_STACK; keep the worker at Tokio's default to catch regressions. + tokio::task::spawn_blocking(move || { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_stack_size(2 * 1024 * 1024) + .enable_all() + .build()? + .block_on(run_volume_sql_worker_e2e(db, language, code)) + }) + .await? +} + +#[cfg(feature = "parquet")] +async fn run_volume_sql_worker_e2e( + db: Pool, + language: ScriptLang, + code: &str, +) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -571,24 +629,11 @@ async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; // 3. Push the job and run with SQL-connected worker - let code = r#"// volume: test-vol data - -import { readFileSync, writeFileSync, existsSync } from "fs"; - -export function main() { - const content = readFileSync("data/hello.txt", "utf-8"); - writeFileSync("data/output.txt", "written by sql worker"); - return { - read_content: content, - output_exists: existsSync("data/output.txt"), - }; -}"#; - let job = JobPayload::Code(RawCode { hash: None, content: code.to_string(), path: None, - language: ScriptLang::Bun, + language, lock: None, cache_ttl: None, cache_ignore_s3_path: None, diff --git a/backend/tests/wac_child_completion_wakes_parent.rs b/backend/tests/wac_child_completion_wakes_parent.rs new file mode 100644 index 0000000000..23c32573ab --- /dev/null +++ b/backend/tests/wac_child_completion_wakes_parent.rs @@ -0,0 +1,189 @@ +//! A WAC v2 parent parks on its dispatched children and is woken by their +//! completions. A child does not always complete through the worker that ran it: +//! the zombie monitor and a force cancel both go straight to +//! `add_completed_job_error`. The parent must be woken from there too, or it sits +//! out its whole suspend window and then runs the task a second time. + +use serde_json::{json, Value}; +use sqlx::{types::Json, Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::{add_completed_job, add_completed_job_error, get_mini_completed_job}; + +const W_ID: &str = "test-workspace"; + +async fn insert_job(db: &Pool, id: Uuid, parent: Option) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner, parent_job) \ + VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', \ + 'script', 'bun', 'u/test-user/wac', 'bun', true, $3)", + ) + .bind(id) + .bind(W_ID) + .bind(parent) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag) \ + VALUES ($1, $2, now(), true, 'bun')", + ) + .bind(id) + .bind(W_ID) + .execute(db) + .await?; + Ok(()) +} + +/// A parent parked on `steps` (step key → child job), the shape +/// `handle_wac_v2_output` leaves behind once the children are pushed. +async fn plant_parked_parent(db: &Pool, steps: &[(&str, Uuid)]) -> anyhow::Result { + let parent = Uuid::new_v4(); + insert_job(db, parent, None).await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 days' \ + WHERE id = $1", + ) + .bind(parent) + .bind(steps.len() as i32) + .execute(db) + .await?; + let job_ids: serde_json::Map = steps + .iter() + .map(|(k, id)| (k.to_string(), json!(id.to_string()))) + .collect(); + let keys: Vec<&str> = steps.iter().map(|(k, _)| *k).collect(); + sqlx::query("INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1, $2)") + .bind(parent) + .bind(json!({ + "_checkpoint": { + "completed_steps": {}, + "pending_steps": { "mode": "dispatch", "keys": keys, "job_ids": job_ids }, + "job_ids": job_ids, + } + })) + .execute(db) + .await?; + for (_, child) in steps { + insert_job(db, *child, Some(parent)).await?; + } + Ok(parent) +} + +async fn parent_state(db: &Pool, parent: Uuid) -> anyhow::Result<(i32, bool, Value)> { + let (suspend, parked, status): (i32, bool, Value) = sqlx::query_as( + "SELECT q.suspend, q.suspend_until IS NOT NULL, s.workflow_as_code_status \ + FROM v2_job_queue q JOIN v2_job_status s USING (id) WHERE q.id = $1", + ) + .bind(parent) + .fetch_one(db) + .await?; + Ok((suspend, parked, status)) +} + +/// The zombie monitor's path: `handle_job_error` → `add_completed_job_error`, never +/// the worker's result processor. The parent must come out of it pullable, with the +/// failure recorded under the step so the workflow's `try/catch` sees a task error. +#[sqlx::test(fixtures("base"))] +async fn a_child_failed_outside_the_worker_wakes_its_parent( + db: Pool, +) -> anyhow::Result<()> { + let child = Uuid::new_v4(); + let parent = plant_parked_parent(&db, &[("slowTask", child)]).await?; + let child_job = get_mini_completed_job(&child, W_ID, &db).await?.unwrap(); + + add_completed_job_error( + &db, + &child_job, + 0, + None, + json!({"name": "ExecutionErr", "message": "Job timed out after no ping"}), + "monitor", + false, + None, + ) + .await?; + + let (suspend, parked, status) = parent_state(&db, parent).await?; + assert_eq!(suspend, 0, "the parent must be released"); + assert!( + parked, + "suspend_until stays set: the suspended pull query keys on it" + ); + let step = &status["_checkpoint"]["completed_steps"]["slowTask"]; + assert_eq!(step["__wmill_error"], json!(true), "{status}"); + assert_eq!(step["child_job_id"], json!(child.to_string())); + assert_eq!( + step["result"]["error"]["message"], + json!("Job timed out after no ping") + ); + assert!( + status["_checkpoint"].get("pending_steps").is_none(), + "nothing left to wait on: {status}" + ); + assert!( + windmill_common::wac::WAC_SUSPEND_READY.swap(false, std::sync::atomic::Ordering::Relaxed) + ); + Ok(()) +} + +/// Only a child the parent is waiting on moves the counter. A child the body +/// launched itself, or a completion arriving after the key was re-dispatched to +/// another job, records its timeline entry and nothing else. +#[sqlx::test(fixtures("base"))] +async fn a_child_the_parent_is_not_waiting_on_leaves_it_parked( + db: Pool, +) -> anyhow::Result<()> { + let awaited = Uuid::new_v4(); + let parent = plant_parked_parent(&db, &[("task", awaited)]).await?; + let stray = Uuid::new_v4(); + insert_job(&db, stray, Some(parent)).await?; + + let stray_job = get_mini_completed_job(&stray, W_ID, &db).await?.unwrap(); + add_completed_job_error( + &db, + &stray_job, + 0, + None, + json!({"message": "boom"}), + "w", + false, + None, + ) + .await?; + + let (suspend, _, status) = parent_state(&db, parent).await?; + assert_eq!( + suspend, 1, + "a stray child must not release the parent: {status}" + ); + assert_eq!(status["_checkpoint"]["completed_steps"], json!({})); + assert!( + status[stray.to_string()]["duration_ms"].is_number(), + "the timeline entry is still stamped: {status}" + ); + + let awaited_job = get_mini_completed_job(&awaited, W_ID, &db).await?.unwrap(); + let result = serde_json::value::to_raw_value(&json!("done"))?; + add_completed_job( + &db, + &awaited_job, + true, + false, + Json(&result), + None, + 0, + None, + false, + None, + false, + ) + .await?; + + let (suspend, _, status) = parent_state(&db, parent).await?; + assert_eq!(suspend, 0); + assert_eq!( + status["_checkpoint"]["completed_steps"]["task"], + json!("done") + ); + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 39b86c59a1..23d88301b7 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -318,19 +318,35 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho resp.text().await? ); } - // ...and the one `settings/global` key on the allowlist, which the CLI reads before - // creating a user on a git-sync push. `ws_base_url` is the control: the handler leaves - // it as ungated as `automate_username_creation`, so only the allowlist stops it. + // ...and the `settings/global` keys on the allowlist, which the CLI reads from a job: on a + // git-sync push, and in `u/admin/hub_sync`. `ws_base_url` is the control: the handler + // leaves it as ungated as these, so only the allowlist stops it. + for key in ["automate_username_creation", "uid", "hub_base_url"] { + let resp = authed( + client().get(format!("{api}/settings/global/{key}")), + &user_wm, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "WM_TOKEN must still read {key}: {}", + resp.text().await? + ); + } + // The same hub pull reads `hub_api_secret` for a private Hub, but a secret stays out of + // a job's reach even when the token borrows a superadmin. let resp = authed( - client().get(format!("{api}/settings/global/automate_username_creation")), - &user_wm, + client().get(format!("{api}/settings/global/hub_api_secret")), + &sa_wm, ) .send() .await?; - assert_eq!( - resp.status(), - 200, - "WM_TOKEN must still read automate_username_creation: {}", + let status = resp.status().as_u16(); + assert!( + status == 401 || status == 403, + "superadmin WM_TOKEN must not read hub_api_secret ({status}): {}", resp.text().await? ); let resp = authed( diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 419be6fdf3..b8bd6b30a7 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3091,16 +3091,16 @@ async fn test_php_job(db: Pool) -> anyhow::Result<()> { let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); - let content = r#" + let content = r#"// schema_validation ) -> anyhow: let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) - .email("test2@windmill.dev") + .as_user("test-user-2", "test2@windmill.dev") .run_until_complete(&db, false, server.addr.port()) .await; @@ -5635,6 +5649,83 @@ async fn test_whileloop_propagates_inner_iterator_eval_failure( Ok(()) } +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_whileloop_skip_if_evaluated_once_at_entry(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Regression test for #11007: `skip_if` on a while-loop module must be + // evaluated once, at loop entry, using the preceding step's result. + // Re-evaluating it on every iteration aliases `results.first` to the + // previous iteration's own result instead, which here lacks `.ok` and + // makes `skip_if` incorrectly turn true after the first iteration. + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "first", + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(): return {\"ok\": True}", + }, + }, + { + "id": "outer", + "value": { + "type": "whileloopflow", + "skip_failures": false, + "modules": [ + { + "id": "inner", + "value": { + "input_transforms": { + "i": { + "type": "javascript", + "expr": "flow_input.iter.index", + }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(i): return i", + }, + }, + ], + }, + "skip_if": { "expr": "!results.first.ok" }, + "stop_after_if": { + "expr": "result >= 2", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job).run_until_complete(&db, false, port).await; + + assert!(cjob.success, "flow should succeed"); + + let outer_module = get_module(&cjob, "outer").expect("outer module status"); + match outer_module { + windmill_common::flow_status::FlowStatusModule::Success { skipped, flow_jobs, .. } => { + assert!( + !skipped, + "while-loop must not be skipped: skip_if should only run once, at entry" + ); + assert_eq!( + flow_jobs.map(|v| v.len()), + Some(3), + "while-loop should run 3 iterations before stop_after_if halts it" + ); + } + other => panic!("expected outer module to be Success, got {other:?}"), + } + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall( diff --git a/backend/tests/ws_specific.rs b/backend/tests/ws_specific.rs index cc15161381..322de598b7 100644 --- a/backend/tests/ws_specific.rs +++ b/backend/tests/ws_specific.rs @@ -391,7 +391,8 @@ async fn test_create_resource_upsert_clears_ws_specific(db: Pool) -> a /// Regression for GHSA-xmr2-98m6-cjf7: a token scoped only to `resources:write:` /// must NOT use the resource-delete cascade to delete a linked secret variable it has -/// no `variables:write` scope for. +/// no `variables:write` scope for. The victim sits at a path the resource owns, which is +/// the only kind the cascade reaches at all. #[sqlx::test(fixtures("ws_specific"))] async fn test_scoped_token_cannot_cascade_delete_linked_variable( db: Pool, @@ -406,7 +407,7 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( "SECRET_TOKEN", ) .json(&json!({ - "path": "u/test-user/victim_secret", + "path": "u/test-user/db_victim_secret", "value": "hunter2", "is_secret": true, "description": "" @@ -421,7 +422,7 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( ) .json(&json!({ "path": "u/test-user/db", - "value": { "password": "$var:u/test-user/victim_secret" }, + "value": { "password": "$var:u/test-user/db_victim_secret" }, "resource_type": "object" })) .send() @@ -443,9 +444,206 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( resp.text().await? ); assert!( - variable_exists(&db, "test-workspace", "u/test-user/victim_secret").await?, + variable_exists(&db, "test-workspace", "u/test-user/db_victim_secret").await?, "victim variable must survive the denied cascade" ); + // The scope check runs after the resource DELETE, so only the rollback keeps the resource + // alive — moving the check out of the transaction would silently delete it on a 403. + let resource_left: Option = + sqlx::query_scalar("SELECT COUNT(*) FROM resource WHERE workspace_id = $1 AND path = $2") + .bind("test-workspace") + .bind("u/test-user/db") + .fetch_one(&db) + .await?; + assert_eq!( + resource_left.unwrap_or(0), + 1, + "the denied delete must roll the resource back too" + ); + + Ok(()) +} + +/// Deleting a resource must not take a variable other things still need. Two gates, each +/// with a way past the other: a variable outside the resource's own path is never its to +/// delete, and even one it owns stays if another resource points at it. +#[sqlx::test(fixtures("ws_specific"))] +async fn test_resource_delete_spares_variables_it_does_not_own( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let create_var = |path: &'static str| { + authed( + client().post(format!("{base}/variables/create")), + "SECRET_TOKEN", + ) + .json(&json!({ "path": path, "value": "hunter2", "is_secret": true, "description": "" })) + .send() + }; + let create_res = |path: &'static str, var: &'static str| { + authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": path, + "value": { "password": format!("$var:{var}") }, + "resource_type": "object" + })) + .send() + }; + + // A shared secret at a path of its own, and two resources reading it. + assert_eq!(create_var("u/test-user/shared_canary").await?.status(), 201); + assert_eq!( + create_res("u/test-user/probe_a", "u/test-user/shared_canary") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/probe_b", "u/test-user/shared_canary") + .await? + .status(), + 201 + ); + + // A secret the resource at the same path owns, which a second resource also reads. + assert_eq!(create_var("u/test-user/owned").await?.status(), 201); + assert_eq!( + create_res("u/test-user/owned", "u/test-user/owned") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/borrower", "u/test-user/owned") + .await? + .status(), + 201 + ); + + for resource in ["u/test-user/probe_a", "u/test-user/owned"] { + let resp = authed( + client().delete(format!("{base}/resources/delete/{resource}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "delete {resource}: {}", + resp.text().await? + ); + } + + assert!( + variable_exists(&db, "test-workspace", "u/test-user/shared_canary").await?, + "a variable the deleted resource only referenced must survive" + ); + assert!( + variable_exists(&db, "test-workspace", "u/test-user/owned").await?, + "an owned variable another resource still references must survive" + ); + + Ok(()) +} + +/// The bulk cascade follows what RLS actually deleted, not what the caller asked for: a +/// resource the request names but leaves standing neither cascades nor stops counting as a +/// referrer. Both halves matter, and neither covers the other. +#[sqlx::test(fixtures("ws_specific"))] +async fn test_bulk_delete_follows_what_rls_deleted(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let create_var = |path: &'static str| { + authed( + client().post(format!("{base}/variables/create")), + "SECRET_TOKEN", + ) + .json(&json!({ "path": path, "value": "hunter2", "is_secret": true, "description": "" })) + .send() + }; + // ws_specific so the flag assertion at the end has something to check. + let create_res = |path: &'static str, var: &'static str| { + authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": path, + "value": { "password": format!("$var:{var}") }, + "resource_type": "object", + "ws_specific": true + })) + .send() + }; + + // Private to test-user: a resource and the secret it owns. + assert_eq!(create_var("u/test-user/hidden_pwd").await?.status(), 201); + assert_eq!( + create_res("u/test-user/hidden", "u/test-user/hidden_pwd") + .await? + .status(), + 201 + ); + // test-user-2's own resource and secret, which the private resource above also reads. + assert_eq!(create_var("u/test-user-2/own_pwd").await?.status(), 201); + assert_eq!( + create_res("u/test-user-2/own", "u/test-user-2/own_pwd") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/reader", "u/test-user-2/own_pwd") + .await? + .status(), + 201 + ); + + // test-user-2 may write the private secret but has no access to its resource at all. + sqlx::query( + "UPDATE variable SET extra_perms = '{\"u/test-user-2\": true}'::jsonb + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/hidden_pwd'", + ) + .execute(&db) + .await?; + + let resp = authed( + client().delete(format!("{base}/resources/delete_bulk")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "paths": ["u/test-user/hidden", "u/test-user-2/own", "u/test-user/reader"] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "bulk delete: {}", resp.text().await?); + + assert!( + variable_exists(&db, "test-workspace", "u/test-user/hidden_pwd").await?, + "the variable of a resource RLS refused to delete must survive" + ); + assert!( + variable_exists(&db, "test-workspace", "u/test-user-2/own_pwd").await?, + "a requested resource RLS left standing still counts as a referrer" + ); + // ws_specific has no RLS policy of its own, so clearing it by requested path rather than + // by deleted path would quietly turn a surviving resource workspace-generic. + assert_eq!( + ws_specific_row_count(&db, "test-workspace", "resource", "u/test-user/hidden").await?, + 1, + "a resource RLS refused to delete must keep its ws_specific flag" + ); Ok(()) } diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 486b08b739..0bb7316dcb 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -767,6 +767,7 @@ impl QueryBuilder for AnthropicQueryBuilder { let AnthropicSSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, events_str, annotations, @@ -790,6 +791,7 @@ impl QueryBuilder for AnthropicQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, @@ -900,6 +902,7 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: None, + reasoning_summary: false, }; AnthropicQueryBuilder::new(AIProvider::Anthropic, platform) diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index cf977f2484..adee6e5053 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -1188,6 +1188,13 @@ impl BedrockQueryBuilder { Some(accumulated_text) }; + // The block folded for replay is also what the reader sees as thinking. Read out + // before the block itself moves into the tool calls below. + let reasoning_text = reasoning + .as_ref() + .and_then(|r| r.reasoning_text.clone()) + .filter(|t| !t.is_empty()); + let tool_calls = streaming_tool_calls_to_openai( accumulated_tool_calls.into_values().collect(), reasoning, @@ -1195,6 +1202,7 @@ impl BedrockQueryBuilder { Ok(ParsedResponse::Text { content, + reasoning: reasoning_text, tool_calls, events_str: if events_str.is_empty() { None diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 21428903fa..8320ef5661 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -666,6 +666,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { let GeminiSSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, mut events_str, stream_event_processor, @@ -698,6 +699,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index 34185a8852..14c9fdb4d3 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -1,6 +1,7 @@ use crate::{ ai_providers::AIProvider, ai_types::OpenAIToolCall, + credentials::ProviderCredentials, image_handler::{prepare_messages_for_api, s3_object_to_content_part}, proxy::{build_openai_compatible_proxy_request, ProxyBuildArgs, ProxyRequest}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, @@ -11,7 +12,14 @@ use crate::{ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{client::AuthedClient, error::Error}; +use std::{ + collections::{BTreeMap, HashMap}, + hash::{DefaultHasher, Hash, Hasher}, + time::{Duration, Instant}, +}; +use windmill_common::{cache::Cache, client::AuthedClient, error::Error}; + +use super::REASONING_OFF_SENTINEL; // Responses API structures #[derive(Deserialize)] @@ -192,13 +200,79 @@ pub struct ResponsesApiTextFormat { pub format: ResponsesApiTextFormatConfig, } -/// Reasoning config for the Responses API (`reasoning: { effort }`). -/// The summary is intentionally not requested, mirroring the copilot chat: OpenAI -/// gates reasoning summaries behind organization verification, so asking for one -/// would fail the request for unverified orgs. +/// Reasoning config for the Responses API (`reasoning: { effort, summary }`). #[derive(Serialize)] pub struct ResponsesApiReasoning { pub effort: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +lazy_static::lazy_static! { + /// Refused reasoning summaries, so later requests skip asking instead of paying a + /// rejected call each. A refusal belongs to the organization the request bills or to the + /// model, so the key holds the model and everything that authenticates (the API key and + /// the resource headers, which can carry it instead). Entries expire as an org gets verified. + static ref REASONING_SUMMARY_UNAVAILABLE: Cache<(String, u64), Instant> = Cache::new(500); +} + +const REASONING_SUMMARY_UNAVAILABLE_TTL: Duration = Duration::from_secs(3600); + +fn reasoning_summary_cache_key( + base_url: &str, + model: &str, + api_key: Option<&str>, + custom_headers: &HashMap, +) -> (String, u64) { + let mut hasher = DefaultHasher::new(); + model.hash(&mut hasher); + api_key.hash(&mut hasher); + // Sorted: two maps with the same entries can iterate them in different orders. + custom_headers + .iter() + .collect::>() + .hash(&mut hasher); + (base_url.to_string(), hasher.finish()) +} + +fn credentials_cache_key(credentials: &ProviderCredentials, model: &str) -> (String, u64) { + reasoning_summary_cache_key( + &credentials.base_url, + model, + credentials.api_key.as_deref(), + &credentials.custom_headers, + ) +} + +/// Whether this model is known to be refused reasoning summaries with these credentials. +pub fn is_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) -> bool { + REASONING_SUMMARY_UNAVAILABLE + .get(&credentials_cache_key(credentials, model)) + .is_some_and(|learned_at| learned_at.elapsed() < REASONING_SUMMARY_UNAVAILABLE_TTL) +} + +/// Record that this model was refused a reasoning summary with these credentials. +pub fn remember_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) { + REASONING_SUMMARY_UNAVAILABLE.insert(credentials_cache_key(credentials, model), Instant::now()); +} + +/// Whether a rejected request was refused over its reasoning summary, e.g. `Your +/// organization must be verified to generate reasoning summaries` (param +/// `reasoning.summary`). An OpenAI-kind resource can also point at a gateway that validates +/// the body strictly and names only the unknown `summary` property. +pub fn rejects_reasoning_summary(status: u16, body: &str) -> bool { + // 422 is how FastAPI-based gateways reject a body that fails validation. + if !matches!(status, 400 | 403 | 422) { + return false; + } + let body = body.to_lowercase(); + let unknown_field = body.contains("additional properties are not allowed") + || body.contains("unrecognized request argument") + || body.contains("extra inputs are not permitted"); + body.contains("reasoning.summary") + || body.contains("verified to generate reasoning summar") + || body.contains("verified to stream reasoning summar") + || (unknown_field && body.contains("summary")) } #[derive(Serialize)] @@ -435,9 +509,13 @@ impl OpenAIQueryBuilder { tools, stream: Some(true), temperature: args.temperature, - reasoning: args - .reasoning_effort - .map(|effort| ResponsesApiReasoning { effort: effort.to_string() }), + reasoning: args.reasoning_effort.map(|effort| ResponsesApiReasoning { + effort: effort.to_string(), + // A request that does not reason has nothing to summarize, yet asking still + // gets an unverified organization's request rejected. + summary: (args.reasoning_summary && effort != REASONING_OFF_SENTINEL) + .then(|| "auto".to_string()), + }), max_output_tokens: args.max_tokens, text, prompt_cache_key: args.prompt_cache_key, @@ -538,6 +616,8 @@ impl QueryBuilder for OpenAIQueryBuilder { } else { Some(parser.accumulated_content) }, + reasoning: (!parser.accumulated_reasoning.is_empty()) + .then_some(parser.accumulated_reasoning), tool_calls: parser.accumulated_tool_calls.into_values().collect(), events_str: Some(parser.events_str), annotations: parser.annotations, @@ -637,8 +717,11 @@ mod tests { } } - async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { - let args = BuildRequestArgs { + fn text_args<'a>( + messages: &'a [OpenAIMessage], + system_prompt: Option<&'a str>, + ) -> BuildRequestArgs<'a> { + BuildRequestArgs { messages, tools: None, model: "gpt-5", @@ -652,14 +735,98 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: Some(PROMPT_CACHE_KEY), - }; + reasoning_summary: true, + } + } + async fn build_body(args: &BuildRequestArgs<'_>) -> String { OpenAIQueryBuilder::new(AIProvider::OpenAI) - .build_request(&args, &client(), "test-workspace") + .build_request(args, &client(), "test-workspace") .await .unwrap() } + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + build_body(&text_args(messages, system_prompt)).await + } + + async fn reasoning_of(effort: Option<&str>, reasoning_summary: bool) -> serde_json::Value { + let messages = vec![message("user", "hi")]; + let args = BuildRequestArgs { + reasoning_effort: effort, + reasoning_summary, + ..text_args(&messages, None) + }; + let request: serde_json::Value = serde_json::from_str(&build_body(&args).await).unwrap(); + request["reasoning"].clone() + } + + #[tokio::test] + async fn requests_a_reasoning_summary_only_when_the_model_reasons() { + assert_eq!( + reasoning_of(Some("high"), true).await, + serde_json::json!({ "effort": "high", "summary": "auto" }) + ); + assert_eq!( + reasoning_of(Some("none"), true).await, + serde_json::json!({ "effort": "none" }) + ); + assert_eq!( + reasoning_of(Some("high"), false).await, + serde_json::json!({ "effort": "high" }) + ); + assert!(reasoning_of(None, true).await.is_null()); + } + + #[test] + fn recognizes_a_refused_reasoning_summary() { + let unverified = r#"{"error":{"message":"Your organization must be verified to generate reasoning summaries. Please go to: https://platform.openai.com/settings/organization/general and click on Verify Organization.","type":"invalid_request_error","param":"reasoning.summary","code":"unsupported_value"}}"#; + assert!(rejects_reasoning_summary(400, unverified)); + assert!(!rejects_reasoning_summary(500, unverified)); + assert!(rejects_reasoning_summary( + 400, + r#"{"detail":"Additional properties are not allowed ('summary' was unexpected)"}"# + )); + assert!(rejects_reasoning_summary( + 422, + r#"{"detail":[{"type":"extra_forbidden","loc":["body","reasoning","summary"],"msg":"Extra inputs are not permitted","input":"auto"}]}"# + )); + assert!(!rejects_reasoning_summary( + 400, + r#"{"error":{"message":"Invalid 'prompt_cache_key': string too long","param":"prompt_cache_key"}}"# + )); + } + + /// A resource can authenticate through `headers` with no API key: one organization's + /// refusal must not withhold summaries from another's. + #[test] + fn keys_a_refused_summary_by_the_header_credential() { + let headers = |pairs: &[(&str, &str)]| { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>() + }; + let url = "https://api.openai.com/v1"; + let org_a = headers(&[("Authorization", "Bearer org-a"), ("X-Trace", "1")]); + let org_a_reordered = headers(&[("X-Trace", "1"), ("Authorization", "Bearer org-a")]); + let org_b = headers(&[("Authorization", "Bearer org-b"), ("X-Trace", "1")]); + + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_b) + ); + assert_eq!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_a_reordered) + ); + // A model can refuse summaries that another model on the same credentials streams. + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5-mini", None, &org_a) + ); + } + /// The worker prepends the system prompt as a system message *and* passes it as /// `system_prompt`; the request must still carry it exactly once. #[tokio::test] diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index bc2d8f2992..b6040d8d79 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -251,6 +251,7 @@ impl QueryBuilder for OtherQueryBuilder { let OpenAISSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, mut events_str, stream_event_processor, @@ -277,6 +278,7 @@ impl QueryBuilder for OtherQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations: Vec::new(), diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index c8f5fbb092..3281b51cd0 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -27,12 +27,17 @@ pub struct BuildRequestArgs<'a> { /// the prefix (the step), never from the request. `None` retries a key the /// endpoint rejected. pub prompt_cache_key: Option<&'a str>, + /// Ask for a summary of the model's reasoning where the provider streams one. + /// `false` once the provider refused summaries to these credentials. + pub reasoning_summary: bool, } /// Response from AI provider pub enum ParsedResponse { Text { content: Option, + /// The thinking the model streamed before the answer, when it emitted any. + reasoning: Option, tool_calls: Vec, events_str: Option, annotations: Vec, diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index 96692d0173..054baeb5de 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -135,6 +135,8 @@ pub trait SSEParser { pub struct OpenAISSEParser { pub accumulated_content: String, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: Box, @@ -146,6 +148,7 @@ impl OpenAISSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), events_str: String::new(), stream_event_processor, @@ -175,6 +178,7 @@ impl SSEParser for OpenAISSEParser { if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { if let Some(delta) = choices.remove(0).delta { if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) { + self.accumulated_reasoning.push_str(&reasoning); let event = StreamingEvent::ReasoningTokenDelta { content: reasoning }; self.stream_event_processor .send(event, &mut self.events_str) @@ -353,6 +357,8 @@ enum ContentBlockState { /// Anthropic SSE Parser for streaming responses pub struct AnthropicSSEParser { pub accumulated_content: String, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: Box, @@ -375,6 +381,7 @@ impl AnthropicSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), events_str: String::new(), stream_event_processor, @@ -455,6 +462,7 @@ impl SSEParser for AnthropicSSEParser { .thinking .get_or_insert_with(String::new) .push_str(&thinking); + self.accumulated_reasoning.push_str(&thinking); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: thinking }, @@ -523,6 +531,7 @@ impl SSEParser for AnthropicSSEParser { .thinking .get_or_insert_with(String::new) .push_str(&thinking); + self.accumulated_reasoning.push_str(&thinking); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: thinking }, @@ -590,6 +599,8 @@ impl SSEParser for AnthropicSSEParser { /// `windmill_ai::ai_google` so the logic can be shared with the API proxy. pub struct GeminiSSEParser { pub accumulated_content: String, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: Box, @@ -603,6 +614,7 @@ impl GeminiSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), events_str: String::new(), stream_event_processor, @@ -621,6 +633,7 @@ impl SSEParser for GeminiSSEParser { }; if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) { + self.accumulated_reasoning.push_str(&reasoning); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: reasoning }, @@ -802,6 +815,14 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.output_text.annotation.added")] AnnotationAdded { annotation: OpenAIUrlCitationEvent }, + /// A new reasoning summary part starts (only sent when `reasoning.summary` was requested) + #[serde(rename = "response.reasoning_summary_part.added")] + ReasoningSummaryPartAdded {}, + + /// Reasoning summary text delta + #[serde(rename = "response.reasoning_summary_text.delta")] + ReasoningSummaryTextDelta { delta: String }, + /// Catch-all for unknown event types #[serde(other)] Other, @@ -810,6 +831,8 @@ pub enum OpenAIResponsesSSEEvent { /// OpenAI Responses API SSE Parser for streaming responses pub struct OpenAIResponsesSSEParser { pub accumulated_content: String, + /// The reasoning summary streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, /// Maps item_id -> (name, call_id) for function calls tool_call_metadata: HashMap, @@ -823,12 +846,15 @@ pub struct OpenAIResponsesSSEParser { pub used_websearch: bool, /// Token usage from response.completed event pub usage: Option, + /// Reasoning summary parts seen so far, to separate them as paragraphs + reasoning_summary_parts: usize, } impl OpenAIResponsesSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), tool_call_metadata: HashMap::new(), tool_call_arguments: HashMap::new(), @@ -837,6 +863,7 @@ impl OpenAIResponsesSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + reasoning_summary_parts: 0, } } } @@ -946,6 +973,28 @@ impl SSEParser for OpenAIResponsesSSEParser { } } + OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => { + self.reasoning_summary_parts += 1; + if self.reasoning_summary_parts > 1 { + self.accumulated_reasoning.push_str("\n\n"); + let event = + StreamingEvent::ReasoningTokenDelta { content: "\n\n".to_string() }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + + OpenAIResponsesSSEEvent::ReasoningSummaryTextDelta { delta } => { + if !delta.is_empty() { + self.accumulated_reasoning.push_str(&delta); + let event = StreamingEvent::ReasoningTokenDelta { content: delta }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + // Ignore other event types OpenAIResponsesSSEEvent::Done {} | OpenAIResponsesSSEEvent::Created {} @@ -1002,6 +1051,33 @@ mod tests { assert_eq!(token_usage.total_tokens, Some(4821)); } + struct ReasoningSink; + + #[async_trait::async_trait] + impl StreamEventSink for ReasoningSink { + async fn send(&self, event: StreamingEvent, events_str: &mut String) -> Result<(), Error> { + if let StreamingEvent::ReasoningTokenDelta { content } = event { + events_str.push_str(&content); + } + Ok(()) + } + } + + #[tokio::test] + async fn streams_openai_responses_reasoning_summary_parts_as_paragraphs() { + let mut parser = OpenAIResponsesSSEParser::new(Box::new(ReasoningSink)); + for data in [ + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":0,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":0,"delta":"**Planning**"}"#, + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":1,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":1,"delta":"Then answer"}"#, + ] { + parser.parse_event_data(data).await.unwrap(); + } + assert_eq!(parser.events_str, "**Planning**\n\nThen answer"); + assert_eq!(parser.accumulated_reasoning, "**Planning**\n\nThen answer"); + } + #[test] fn openai_delta_parses_reasoning_content() { // DeepSeek and similar stream reasoning under `reasoning_content`. diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 6d231dc675..ae85718e8a 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -103,6 +103,7 @@ struct AIAgentArgsRaw { streaming: Option, max_iterations: Option, memory: Option, + enabled_tools: Option>, // Legacy field for backward compatibility messages_context_length: Option, #[serde(default)] @@ -123,6 +124,9 @@ pub struct AIAgentArgs { pub streaming: Option, pub max_iterations: Option, pub memory: Option, + /// Which of the agent's tools this run may call; `narrow_roster` holds what the names are and + /// what `None` means. + pub enabled_tools: Option>, pub credentials_check: bool, } @@ -155,6 +159,7 @@ impl From for AIAgentArgs { streaming: raw.streaming, max_iterations: raw.max_iterations, memory, + enabled_tools: raw.enabled_tools, credentials_check: raw.credentials_check.unwrap_or(false), } } @@ -369,6 +374,12 @@ pub struct AIAgentResult<'a> { pub messages: Vec>, #[serde(skip_serializing_if = "Option::is_none")] pub wm_stream: Option, + /// The model's thinking across every iteration of the loop, in order, blank-line + /// separated. Present whenever the provider's parser surfaced any, whether or not + /// the step streams, so a downstream step never has to pick it out of `wm_stream`. + /// Absent when the model thought nothing. + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] pub usage: Option, } diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 09720b2203..89e3c7c6b2 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -6,6 +6,7 @@ use axum::{ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::Row; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use windmill_common::{ assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind}, db::UserDB, @@ -13,7 +14,9 @@ use windmill_common::{ utils::escape_ilike_pattern, }; -use windmill_api_auth::{build_scope_path_predicate, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_filter, build_scope_path_predicate, ApiAuthed, ScopePathFilter, +}; // Partition-range backfill preview. The logic (producer resolution, range // enumeration, status join) is enterprise: the `private` build compiles the @@ -33,6 +36,7 @@ pub fn workspaced_service() -> Router { .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) .route("/graph", get(asset_graph)) + .route("/column_lineage", get(dbt_column_lineage)) .route("/pipelines", get(list_pipeline_folders)) .route("/partitions", get(list_partitions)) .route("/partitions_in_range", get(list_partitions_in_range)) @@ -663,10 +667,21 @@ struct DbtAssetProvenance { description: Option, #[serde(skip_serializing_if = "Vec::is_empty")] data_tests: Vec, - /// Declared column metadata (name -> description). NOT column lineage — - /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14). + /// Declared column metadata (name -> description): what `manifest.json` + /// carries, which is only the columns an author wrote down. #[serde(skip_serializing_if = "Option::is_none")] columns: Option, + /// Every column of the relation, typed and in order — + /// `[{"name": …, "type": …}]` — from the engine's static analysis. Present + /// only for a project that opted into it. + /// + /// Gated exactly like `columns` and the model's SQL: a full column list is + /// the shape of what the author WROTE, one level finer than the `ref()` + /// graph, which is ungated only because it draws relations the caller + /// already sees in `asset`. Widening that boundary has to be a decision, not + /// a consequence of a project turning the analysis pass on. + #[serde(skip_serializing_if = "Option::is_none")] + column_schema: Option, /// A source's declared freshness policy, for the staleness chip. #[serde(skip_serializing_if = "Option::is_none")] freshness: Option, @@ -946,6 +961,458 @@ struct DbtLineageEdge { to_asset_path: String, } +/// One column-to-column edge, in the same terms: the two relations and the two +/// columns, never dbt's node ids. +#[derive(Serialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct DbtColumnLineageEdge { + from_asset_path: String, + from_column: String, + to_asset_path: String, + to_column: String, + /// dbt's own word for how the value travelled: `copy` (passthrough), `mod` + /// (transformed), `scan` (read to produce the ROW rather than the value — a + /// join key, a predicate, a `group by`). Sent verbatim, including a kind + /// this engine version invented, because the renderer decides what a kind + /// means and the set is the engine's. + kind: String, +} + +/// The dbt relations a view is tracing, and which stored graph to read them +/// from. +/// +/// Several relations, answered as one union, because ONE selection reaches +/// several: a script's output column can be derived from columns of several dbt +/// models, and a model's columns can be consumed by scripts that feed others. +/// Asking per relation instead is a request per boundary plus the bookkeeping to +/// stitch the answers together and decide which of them is still current — which +/// is a cache, and is what taking them together exists to not need. +pub struct ColumnLineageQuery { + /// The `dbt://` relations whose lineage to return. + pub asset_paths: Vec, + /// The deployed version a view is drawing, when it is drawing one — the dbt + /// editor, which shows a single project as of a single deploy. + /// + /// A version-pinned answer is that version's project ALONE, the same as a + /// job-pinned one: the pin exists so the trace describes the stored graph on + /// screen, and another project's live graph is not part of it. Only the + /// unpinned answer crosses projects. + /// + /// A run's or an editor buffer's graph is NOT reachable from here: it pins + /// to a job, and that costs the job-read gate. + pub dbt_script_hash: Option, +} + +impl ColumnLineageQuery { + /// Built from the raw pairs rather than deserialized as a struct, because + /// `asset_path` REPEATS and `serde_urlencoded` — what `Query` deserializes + /// with — reads no sequence from a repeated key. A GET rather than a POST + /// body carrying the list: the method decides a scoped token's action, so a + /// POST would ask `assets:write` for a read and refuse a read-only token + /// outright. + pub fn from_query_pairs(pairs: Vec<(String, String)>) -> windmill_common::error::Result { + let mut asset_paths: Vec = Vec::new(); + let mut dbt_script_hash = None; + for (key, value) in pairs { + match key.as_str() { + "asset_path" => asset_paths.push(value), + // Hex, like every other script-hash parameter, so a page can + // pass `job.script_hash` verbatim. + "dbt_script_hash" => { + dbt_script_hash = + Some(serde_json::from_value(Value::String(value)).map_err(|_| { + windmill_common::error::Error::BadRequest( + "dbt_script_hash is not a script hash".to_string(), + ) + })?) + } + _ => {} + } + } + // REFUSED, not answered empty. A caller that named no relation — or + // misspelled the parameter — asked for something, and an empty component + // is what a relation with no lineage returns, so answering that way says + // "this has none" for a question that was never asked. + if asset_paths.is_empty() { + return Err(windmill_common::error::Error::BadRequest( + "at least one asset_path is required".to_string(), + )); + } + if asset_paths.len() > MAX_ASKED_RELATIONS { + return Err(windmill_common::error::Error::BadRequest(format!( + "at most {MAX_ASKED_RELATIONS} asset_path values may be asked about at once" + ))); + } + Ok(ColumnLineageQuery { asset_paths, dbt_script_hash }) + } +} + +#[derive(Serialize, Debug, PartialEq, Eq)] +pub struct ColumnLineageResponse { + /// Direct (`copy` / `mod`) column edges of the component the asked-for + /// relations' columns sit in, in the terms the canvas draws. Empty when no + /// project involved asked for the analysis pass, which is the ordinary case. + edges: Vec, + /// The component reaches further than what is here: `edges` holds the part + /// nearest the asked-for relations. Said rather than silently cut, because a + /// trace that stops short is otherwise indistinguishable from one that ends. + truncated: bool, +} + +/// How many edges one trace may carry back. The renderer draws a box per column, +/// so a component past this is unreadable however it is served — a synthetic +/// 3000-model project whose models share a column returns 58k direct edges and +/// 7.3MB. Applied over the walk, which is the LAST filter, so what survives is +/// the part nearest the selection rather than an arbitrary slice of it. +const MAX_TRACE_EDGES: usize = 5_000; + +/// How many times one trace may discover a project it has not read yet. Each +/// round costs a gate and a fetch, and a component crossing this many projects +/// has already outgrown what the canvas can show; stopping says what the edge +/// bound says. +const MAX_OWNER_ROUNDS: usize = 8; + +/// How many edges a trace may already hold before it stops looking for projects +/// it has not read. NOT a bound on what one trace fetches: the seeds' own +/// projects are read whole whatever their size, because reading them IS the +/// answer, and a project's edges arrive whole in any case — the walk is what +/// decides which of them are in the component, so a `LIMIT` would cut a set that +/// need not contain the asked-for relation at all. What a single fetch is +/// bounded by is the ingest's `MAX_COLUMN_EDGES` per version. This bounds the +/// EXPANSION on top of that: reading a further project's worth once a trace is +/// already this size buys nothing the walk will not cut at `MAX_TRACE_EDGES`. +const EXPANSION_EDGE_BUDGET: usize = 100_000; + +/// How many relations one request may ask about. Generous: the pipeline page +/// sends every dbt relation the selection's own producer lineage reaches, which +/// is a handful even in a large folder. It exists so a crafted request cannot +/// hand `= ANY($2)` an arbitrarily long array. +const MAX_ASKED_RELATIONS: usize = 1_000; + +/// A stored project graph: a deployed version, or one job's snapshot of it. +/// `script_hash` is NULL for an editor buffer's parse, which names no version. +type ProjectVersion = (String, Option, uuid::Uuid); + +async fn dbt_column_lineage( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(pairs): Query>, +) -> JsonResult { + // `None`: pinning to one run is job-scoped and this endpoint is authorized + // as `assets:read`. See `dbt_column_lineage_for`. + let q = ColumnLineageQuery::from_query_pairs(pairs)?; + dbt_column_lineage_for(&authed, &w_id, user_db, q, None).await +} + +/// The column-level lineage the asked-for relations sit in, optionally as one +/// run saw it. +/// +/// AUTHORIZES NOTHING BY ITSELF, on the same contract as `asset_graph_for`: +/// `assets:read` always, and the job-read gate for `Some(pinned)`, whose path +/// and hash are then taken from that job's row rather than from the caller. +/// +/// A column trace is transitive and a relation is not owned by one project, so +/// the answer grows a project at a time: resolve who owns the relations reached +/// so far, read their edges, walk, and repeat for the relations that walk newly +/// reached. Every round re-applies the caller's gate to the projects it +/// discovers — a relation being reachable from a project the caller may read +/// says nothing about the project on the other side of it. +pub async fn dbt_column_lineage_for( + authed: &ApiAuthed, + w_id: &str, + user_db: UserDB, + q: ColumnLineageQuery, + pinned: Option, +) -> JsonResult { + // A column-level view is the shape of what the author WROTE, so it takes the + // model's own gate rather than the relation's. + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(authed, "scripts", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + let (pinned_path, script_hash) = match pinned.as_ref() { + // The job's own version, so a pin cannot name one project's run while + // claiming another's version — including when it names NONE, which is + // the editor buffer. + Some(p) => (Some(p.script_path.as_str()), p.script_hash), + None => (None, q.dbt_script_hash.map(|h| h.0)), + }; + let pinned_job_id = pinned.as_ref().map(|p| p.job_id); + + let seeds: BTreeSet = q.asset_paths.into_iter().collect(); + let mut tx = user_db.begin(authed).await?; + + // Relations whose owners have been asked for, project graphs already read, + // and the edges they yielded. These are what end the loop: a round asks only + // about relations not asked about before and reads only projects not read + // before, so it stops as soon as one of the two runs out. + let mut asked: HashSet = HashSet::new(); + let mut read: HashSet = HashSet::new(); + let mut edges: Vec = Vec::new(); + let mut answer: Vec = Vec::new(); + let mut pending: Vec = seeds.iter().cloned().collect(); + let mut truncated = false; + let mut rounds = 0usize; + + loop { + if pending.is_empty() { + break; + } + // Which project version owns each of these relations, under this + // caller's access. Usually one row per relation; a relation a second + // project declares as a source has two, and each answers for its own + // lineage. + let owners = sqlx::query!( + r#"SELECT DISTINCT n.script_path AS "script_path!", n.script_hash, n.job_id AS "job_id!" + FROM dbt_node n + WHERE n.workspace_id = $1 AND n.asset_path = ANY($2) + -- The run's snapshot, or the deployed graph when that job stored + -- none -- a build pins only if it wrote one. + AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $5) + THEN $5::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END + -- The gate, re-decided for every project the walk reaches. That + -- is what resolving owners in a loop is for: being entitled to + -- one project is not being entitled to the one that declares a + -- relation it hands over. + AND ( $6 + OR n.script_path = ANY($7) + OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx + WHERE n.script_path = pfx + OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) ) + AND CASE + -- Pinned: which version comes from a job this caller was + -- already granted, so `script` does not decide THAT -- but + -- it still decides whether the project may be read, the + -- same second gate `script_visible` is on the graph. Being + -- entitled to a run is not being entitled to the SQL + -- behind it, and column lineage is that SQL's shape. A + -- version-less row is exempt because it is an editor + -- buffer, which has no `script` row to ask and reaches + -- this only through the parse job that wrote it. + -- + -- One project answers, so a pinned trace never crosses + -- into another: neither does the graph it annotates. + WHEN $4::text IS NOT NULL + THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint + AND ($3::bigint IS NULL OR EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3)) + -- A named version: the deployed one an editor is drawing. + -- `script` is read under RLS, so this is the visibility + -- check as well as the existence one. A hash names one + -- script row, so this arm answers for one project too — + -- and deliberately: a pin says which stored graph is on + -- screen, and another project's live graph is not it. + WHEN $3::bigint IS NOT NULL + THEN n.script_hash = $3 AND EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3) + -- Otherwise the version deployed now: an older one's rows + -- outlive it in `dbt_node` until the sweep, and describe a + -- project that is no longer what runs. `language` narrows + -- it the way the graph's own resolution does, so a path + -- that has since become a script of another kind draws and + -- explains the same version rather than disagreeing. Read + -- under RLS, so a project the caller cannot see resolves + -- to NULL and matches nothing. + ELSE n.script_hash = ( + SELECT sc.hash FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.language = 'dbt' + AND sc.deleted = false AND sc.archived = false + ORDER BY sc.created_at DESC LIMIT 1) + END"#, + w_id, + &pending[..], + script_hash, + pinned_path, + pinned_job_id, + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + asked.extend(pending.drain(..)); + + let fresh: Vec = owners + .into_iter() + .map(|o| (o.script_path, o.script_hash, o.job_id)) + .filter(|k| read.insert(k.clone())) + .collect(); + // Nothing left this caller may read and has not read: the component is + // whole, however many relations were still waiting to be asked about. + // Their owners are projects already in hand. + if fresh.is_empty() { + break; + } + // From here a project exists that this answer will not contain, so the + // two stops below are cuts and are reported as such. Deciding it after + // the owners query rather than before is what keeps a big project's + // small component from being called truncated: `pending` alone only says + // a relation has not been ASKED about, not that anything was left out. + rounds += 1; + if rounds > MAX_OWNER_ROUNDS || edges.len() >= EXPANSION_EDGE_BUDGET { + truncated = true; + break; + } + let fresh_paths: Vec = fresh.iter().map(|k| k.0.clone()).collect(); + let fresh_hashes: Vec> = fresh.iter().map(|k| k.1).collect(); + let fresh_jobs: Vec = fresh.iter().map(|k| k.2).collect(); + + // DIRECT kinds only. `scan` -- the column was read to produce the ROW, + // not the value -- reaches every output column of its model, so it is + // most of a project's stored lineage and none of what a trace draws. + // It stays in the table for a later view to ask for. + let rows = sqlx::query!( + r#"SELECT p.asset_path AS "from_path!", e.parent_column AS "from_column!", + c.asset_path AS "to_path!", e.child_column AS "to_column!", + e.lineage_kind AS "kind!" + FROM unnest($2::text[], $3::bigint[], $4::uuid[]) + AS o(script_path, script_hash, job_id) + JOIN dbt_column_edge e ON e.workspace_id = $1 + AND e.script_path = o.script_path + AND e.job_id = o.job_id + -- `=` still, with the NULL-to-NULL case + -- spelled out and gated on the pin: a + -- version-less row's hash is NULL on both + -- sides, which `=` never matches, but + -- `IS NOT DISTINCT FROM` would cost the + -- equality its index bound everywhere else. + AND (e.script_hash = o.script_hash + OR ($5::text IS NOT NULL + AND o.script_hash IS NULL + AND e.script_hash IS NULL)) + JOIN dbt_node p ON p.workspace_id = e.workspace_id + AND p.script_path = e.script_path + AND p.script_hash IS NOT DISTINCT FROM e.script_hash + AND p.job_id = e.job_id + AND p.unique_id = e.parent_unique_id + JOIN dbt_node c ON c.workspace_id = e.workspace_id + AND c.script_path = e.script_path + AND c.script_hash IS NOT DISTINCT FROM e.script_hash + AND c.job_id = e.job_id + AND c.unique_id = e.child_unique_id + WHERE e.lineage_kind IN ('copy', 'mod') + AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL"#, + w_id, + &fresh_paths[..], + &fresh_hashes[..] as &[Option], + &fresh_jobs[..], + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + edges.extend(rows.into_iter().map(|r| DbtColumnLineageEdge { + from_asset_path: r.from_path, + from_column: r.from_column, + to_asset_path: r.to_path, + to_column: r.to_column, + kind: r.kind, + })); + // Two projects can describe one relation, so the same edge can arrive + // twice. Sorted as well as deduplicated: the walk reads the incidence + // lists in this order, so the answer does not depend on which round a + // project was discovered in. + edges.sort(); + edges.dedup(); + + let walked = component(&edges, &seeds); + answer = walked.edges; + truncated = walked.truncated; + if truncated { + break; + } + // The relations the walk newly reached. Their owners are the next round's + // question: this project declares them, and so may another. + pending = answer + .iter() + .flat_map(|e| [&e.from_asset_path, &e.to_asset_path]) + .filter(|p| !asked.contains(*p)) + .cloned() + .collect::>() + .into_iter() + .collect(); + } + tx.commit().await?; + Ok(Json(ColumnLineageResponse { truncated, edges: answer })) +} + +struct WalkedComponent { + edges: Vec, + truncated: bool, +} + +/// The edges of the connected component the asked-for relations sit in, nearest +/// first and at most `MAX_TRACE_EDGES` of them. +/// +/// The canvas lays out the component of the selected relation's columns, so a +/// project's other model families are edges nothing it draws can reach. Walked +/// here rather than in SQL: a recursive CTE has no index to walk, so it rescans +/// the whole edge set once per level — measured at 1.24s against 59ms for the +/// query alone on a 3000-model project, for a walk that is microseconds over a +/// map. Columns are keyed by relation, not by project, which is how the canvas +/// keys them too: two projects describing one relation draw one node. +/// +/// Breadth-first, so the bound cuts the far end of the trace rather than an +/// arbitrary part of it. +fn component(edges: &[DbtColumnLineageEdge], seeds: &BTreeSet) -> WalkedComponent { + let mut incident: HashMap<(&str, &str), Vec> = HashMap::new(); + for (i, e) in edges.iter().enumerate() { + incident + .entry((&e.from_asset_path, &e.from_column)) + .or_default() + .push(i); + incident + .entry((&e.to_asset_path, &e.to_column)) + .or_default() + .push(i); + } + let mut start: Vec<(&str, &str)> = incident + .keys() + .filter(|(path, _)| seeds.contains(*path)) + .copied() + .collect(); + start.sort(); + let mut seen_node: HashSet<(&str, &str)> = start.iter().copied().collect(); + let mut queue: VecDeque<(&str, &str)> = start.into(); + let mut taken = vec![false; edges.len()]; + let mut kept: Vec = Vec::new(); + let mut truncated = false; + 'walk: while let Some(node) = queue.pop_front() { + for &i in incident.get(&node).map(Vec::as_slice).unwrap_or_default() { + if std::mem::replace(&mut taken[i], true) { + continue; + } + if kept.len() == MAX_TRACE_EDGES { + truncated = true; + break 'walk; + } + kept.push(i); + let e = &edges[i]; + let ends = [ + (e.from_asset_path.as_str(), e.from_column.as_str()), + (e.to_asset_path.as_str(), e.to_column.as_str()), + ]; + for end in ends { + if seen_node.insert(end) { + queue.push_back(end); + } + } + } + } + // Back into edge order, so a response does not carry the walk's shape. + kept.sort_unstable(); + WalkedComponent { edges: kept.into_iter().map(|i| edges[i].clone()).collect(), truncated } +} + async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, @@ -1323,7 +1790,7 @@ pub async fn asset_graph_for( n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path, n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description, n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node, - n.columns, n.freshness, + n.columns, n.column_schema, n.freshness, n.raw_code, n.original_file_path, -- Whether the caller may read the project this row describes. -- The query deliberately reaches outside the requested folder @@ -1379,6 +1846,10 @@ pub async fn asset_graph_for( // `ref()` lineage between two models, resolved to the relations they // produce. Joined to `dbt_node` on both key columns because a dbt // `unique_id` is only unique within its project. + // + // Column lineage is NOT here. It is stored per relation and per column, and + // this response is folder-wide and polled by a run page, so it carries only + // what the canvas draws for every node at once. let dbt_edge_rows = sqlx::query!( r#"WITH live AS ( SELECT * FROM ( @@ -1604,6 +2075,7 @@ pub async fn asset_graph_for( description: r.description.clone().filter(|_| source_allowed), data_tests: vec![], columns: r.columns.clone().filter(|_| source_allowed), + column_schema: r.column_schema.clone().filter(|_| source_allowed), freshness: r.freshness.clone().filter(|_| source_allowed), }; // One relation can carry rows from several projects — typically a model diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs index f2bb98a46c..ebeee49ecc 100644 --- a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -8,7 +8,9 @@ //! whole dbt half, and every later fix in this area re-touched one of the two. use sqlx::{Pool, Postgres}; -use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun}; +use windmill_api_assets::{ + asset_graph_for, dbt_column_lineage_for, ColumnLineageQuery, GraphQuery, PinnedRun, +}; use windmill_api_auth::ApiAuthed; use windmill_common::db::UserDB; @@ -84,6 +86,34 @@ async fn seed(db: &Pool, job: uuid::Uuid) { .execute(db) .await .unwrap(); + // The relation it reads, and the column edge between them. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders', + 'u/a/wh/analytics/raw_orders', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', + 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); // A test node, for the arguments it carries: `accepted_values` spells out a // column's domain. sqlx::query!( @@ -365,7 +395,25 @@ async fn seed_editor_graph(db: &Pool, job: uuid::Uuid) { r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, raw_code, tags) VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft', - 'u/a/wh/analytics/draft', 'select 3', '{}')"#, + 'u/a/wh/analytics/draft', 'select 3', '{}'), + ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src', + 'u/a/wh/analytics/draft_src', 'select 4', '{}')"#, + WS, + PATH, + job + ) + .execute(db) + .await + .unwrap(); + // A version-less row's `script_hash` is NULL on both sides of every join and + // every visibility check, and `= NULL` is never true — so the column edges + // need the same NULL arm the node query has, or a buffer parse renders its + // columns and none of their lineage. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", WS, PATH, job @@ -451,3 +499,515 @@ async fn an_editor_graph_renders_only_through_its_own_job(db: Pool) { "nor of a run of the deployed version: {deployed_run}" ); } + +async fn column_lineage_q( + db: &Pool, + authed: &ApiAuthed, + pairs: Vec<(String, String)>, + pinned: Option, +) -> serde_json::Value { + let res = dbt_column_lineage_for( + authed, + WS, + UserDB::new(db.clone()), + ColumnLineageQuery::from_query_pairs(pairs).unwrap(), + pinned, + ) + .await + .unwrap(); + serde_json::to_value(&res.0).unwrap() +} + +async fn column_lineage( + db: &Pool, + authed: &ApiAuthed, + asset_paths: &[&str], + pinned: Option, +) -> serde_json::Value { + let pairs = asset_paths + .iter() + .map(|p| ("asset_path".to_string(), p.to_string())) + .collect(); + column_lineage_q(db, authed, pairs, pinned).await +} + +/// The buffer parse's own lineage, which is the case the versionless rows exist +/// for. Its `script_hash` is NULL on both sides of every join and every +/// visibility check, and `= NULL` is never true — so the versionless arm has to +/// be written for it, or a parse renders its columns and none of their lineage. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_editor_buffers_column_lineage_answers_through_its_job(db: Pool) { + let parse = uuid::Uuid::from_u128(9); + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_editor_graph(&db, parse).await; + let admin = ApiAuthed { is_admin: true, ..outsider() }; + + let pinned = PinnedRun { job_id: parse, script_path: PATH.to_string(), script_hash: None }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], Some(pinned)).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/draft_src", + "from_column": "raw", + "to_asset_path": "u/a/wh/analytics/draft", + "to_column": "clean", + "kind": "mod", + }]), + ); + // Unpinned, the same relation resolves through the deployed version, which + // never heard of the buffer's models. + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], None).await["edges"], + serde_json::json!([]), + "a buffer's lineage is reachable only through the job that parsed it" + ); +} + +/// Being entitled to a RUN is not being entitled to the SQL behind it, and +/// column lineage is that SQL's shape. The pinned graph draws the relations for +/// a share-link viewer and redacts what the author wrote; the lineage is the +/// second, and resolving the version from the job must not be mistaken for +/// deciding that too. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_pinned_run_does_not_hand_over_the_projects_column_lineage(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + + assert_eq!( + column_lineage( + &db, + &outsider(), + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([]), + "the run renders for them, its column-level shape does not" + ); + assert_eq!( + column_lineage( + &db, + &ApiAuthed { is_admin: true, ..outsider() }, + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "while a reader of the project gets it" + ); +} + +/// The deployed version of the same two models, plus a `scan` edge beside the +/// direct one: `seed`'s rows are a run's snapshot, and the unpinned answer is +/// the version's own graph. +async fn seed_deployed_orders(db: &Pool) { + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders', + 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders', + 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); +} + +/// A column-level view is the shape of what the author WROTE, so it takes the +/// script's own gate — the same one that keeps `raw_code` behind access to the +/// project. `scan` says the column was read to produce the ROW rather than the +/// value, so it reaches every output column of its model and is never served. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_takes_the_scripts_gate_and_only_the_direct_kinds(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the direct edge, and not the `scan` one beside it" + ); + assert_eq!( + column_lineage(&db, &outsider(), &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([]), + "and nothing at all for a caller who cannot read the project" + ); +} + +/// One project routinely holds model families that share no column, and the +/// canvas lays out the connected component of the selected relation's columns. +/// Answering with the project's other components sends edges nothing can draw. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_stops_at_the_selected_relations_component(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + // A second family in the same project version, reaching neither of the two + // relations `seed` wired together. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock', + 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily', + 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], Some(pinned())).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the orders family, and not the stock one beside it in the same project" + ); + assert_eq!( + column_lineage( + &db, + &admin, + &["u/a/wh/analytics/stock_daily"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + "and the other way round — reached from the child end, which is upstream" + ); +} + +/// A deployed dbt project in `folder`, declaring `parent`'s relation as a source +/// and deriving `child` from it. `orders → mart → secret_out` is three projects +/// chained through two shared relations. +async fn seed_neighbour_project( + db: &Pool, + folder: &str, + hash: i64, + parent: (&str, &str), + child: (&str, &str), +) { + let path = format!("f/{folder}/proj"); + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) + VALUES ($1, $2, $2, '{}', '{}')", + WS, + folder + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + hash, + path, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4, + 'source', $4, $5, '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6, + 'model', $6, $7, '{}')", + WS, + path, + hash, + parent.0, + parent.1, + child.0, + child.1, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + WS, + path, + hash, + parent.0, + child.0, + ) + .execute(db) + .await + .unwrap(); +} + +/// A trace crosses out of the project it started in, and the gate crosses with +/// it. +/// +/// A relation one project produces is another's source, so the component reaches +/// edges the first project's owner set never named — that is what resolving +/// owners to a fixpoint is for. The other half is that the caller's access has +/// to be re-decided for each project discovered on the way: reaching a relation +/// says nothing about who may read the project on the far side of it. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_crosses_projects_only_where_the_caller_may_read_them(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + seed_neighbour_project( + &db, + "mid", + 43, + ("orders", "u/a/wh/analytics/orders"), + ("mart", "u/a/wh/analytics/mart"), + ) + .await; + seed_neighbour_project( + &db, + "secret", + 44, + ("mart", "u/a/wh/analytics/mart"), + ("secret_out", "u/a/wh/analytics/secret_out"), + ) + .await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let reached = |body: &serde_json::Value| { + body["edges"] + .as_array() + .unwrap() + .iter() + .map(|e| e["to_asset_path"].as_str().unwrap().to_string()) + .collect::>() + }; + + let all = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&all), + [ + "u/a/wh/analytics/mart", + "u/a/wh/analytics/orders", + "u/a/wh/analytics/secret_out" + ] + .map(String::from) + .into(), + "two projects out from the one asked about, not one: {all}" + ); + + // Granted the first two folders and not the third. The edges of the project + // they may read are theirs; the one beyond it is not, even though the + // relation joining them is in the answer. + let partial = ApiAuthed { + folders: vec![ + ("private".to_string(), false, false), + ("mid".to_string(), false, false), + ], + ..outsider() + }; + let some = column_lineage(&db, &partial, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&some), + ["u/a/wh/analytics/mart", "u/a/wh/analytics/orders"] + .map(String::from) + .into(), + "the trace stops where the caller's access does: {some}" + ); + + // The other half of the same gate, and the half that is hand-written SQL + // rather than RLS: a token scoped to one folder reaches the projects in it + // and no others, whatever its grants say. + let scoped = ApiAuthed { + is_admin: true, + scopes: Some(vec!["scripts:read:f/private/*".to_string()]), + ..outsider() + }; + let scoped = column_lineage(&db, &scoped, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&scoped), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "and where its scope does: {scoped}" + ); + + // A version pin answers for that version's project alone — the dbt editor, + // which draws one project as of one deploy. Crossing into `mid` here would + // annotate that canvas with relations it does not draw. + let pinned_version = column_lineage_q( + &db, + &admin, + vec![ + ( + "asset_path".to_string(), + "u/a/wh/analytics/orders".to_string(), + ), + ("dbt_script_hash".to_string(), format!("{:016x}", HASH)), + ], + None, + ) + .await; + assert_eq!( + reached(&pinned_version), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "a version pin does not cross into the project beside it: {pinned_version}" + ); +} + +/// The bound on the answer, and that hitting it is said rather than silently +/// cut: a trace that stops reads exactly like one that ends. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_component_past_the_bound_is_cut_and_says_so(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // One direct edge per column pair, more of them than a trace may carry. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 6000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!(body["edges"].as_array().unwrap().len(), 5000); + assert_eq!(body["truncated"], serde_json::json!(true)); +} + +/// The other side of the same flag: a project big enough to stop the expansion +/// still answers a small component WHOLE, and must not claim it was cut. +/// +/// Reaching the size that stops expansion says nothing on its own — nor does a +/// relation whose owners have not been asked about, since those owners are +/// usually the project already in hand. Only a project this caller may read and +/// this answer does not contain is a cut. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_whole_component_in_a_big_project_is_not_called_cut(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // A second family, two relations and one edge, sharing no column with the + // first — the whole of what a selection on it should return. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock', + 'model', 'stock', 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily', + 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + // And enough unrelated families beside them to pass the expansion budget. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 100000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/stock_daily"], None).await; + assert_eq!( + body["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + ); + assert_eq!(body["truncated"], serde_json::json!(false)); +} diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index b9edde3c6d..04513b4c05 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -1053,6 +1053,10 @@ pub async fn fetch_api_authed_from_permissioned_as( db: &DB, username_override: Option, ) -> error::Result { + // Keyed by the supplied address, so an entry built for a principal's previous holder is reused + // while that address is still supplied, until its 120s expiry: a cached dispatch address is + // evicted sooner, an app's stored one (a username deleted then reused) may not be. Accepted; + // the rebuild after expiry is the current holder's. let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); let mut api_authed = match API_AUTHED_CACHE.get(&key) { @@ -1068,7 +1072,10 @@ pub async fn fetch_api_authed_from_permissioned_as( let api_authed = ApiAuthed { username: authed.username, - email, + // The resolved one, not the address we were handed: that is the point of + // `fetch_authed_from_permissioned_as` validating it against the principal's live + // binding, and this value goes on to the job row, `job_perms` and the JWT. + email: authed.email, is_admin: authed.is_admin, is_operator: authed.is_operator, groups: authed.groups, diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 47ea12c04b..29ee0fe36a 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -1073,12 +1073,15 @@ fn scope_grants_access( /// the caller's own row; `email` and `allowed_domain_auto_invite` are derived from the /// token itself and touch no table. /// -/// `settings/global/automate_username_creation` is the one instance setting on the list. -/// `get_global_setting` exempts a handful of keys from its own super-admin gate, that one -/// among them, so the boolean is already readable by every authenticated user; it is here -/// because the CLI reads it before creating a user during a git-sync push, which runs as a -/// job. The other ungated keys have no such caller, so they stay confined — being ungated -/// earns a key nothing on its own. +/// Three instance settings are on the list. `get_global_setting` exempts a handful of keys +/// from its own super-admin gate, these among them, so each is already readable by every +/// authenticated user; each is here because the CLI reads it from a job: +/// `automate_username_creation` before creating a user during a git-sync push, `uid` and +/// `hub_base_url` when `u/admin/hub_sync` pulls resource types from the Hub. The other +/// ungated keys have no such caller, so they stay confined — being ungated earns a key +/// nothing on its own. Listing a gated key earns it nothing either: `require_super_admin` +/// refuses every job token, so `hub_api_secret`, which that pull reads for a private Hub, +/// stays out of a job's reach whatever this list says. /// /// Deliberately absent, as each crosses that line: `users/list_invites` (returns the /// workspace ids the identity was invited to), `users/tokens/list` (credential metadata @@ -1096,6 +1099,8 @@ fn is_global_read_open_to_job_token(route_path: &str) -> bool { | "/api/users/tutorial_progress" | "/api/workspaces/allowed_domain_auto_invite" | "/api/settings/global/automate_username_creation" + | "/api/settings/global/uid" + | "/api/settings/global/hub_base_url" | "/api/docs/search" | "/api/docs/page" | "/api/integrations/hub/list" diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 7ad99996d5..7598021f00 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -59,14 +59,116 @@ struct Config { config: serde_json::Value, } +/// Credential-bearing fields across the `ObjectSettings` variants, which are flattened into a +/// single object by the `type` tag. +const OBJECT_STORE_SECRET_KEYS: &[&str] = + &["access_key", "secret_key", "accessKey", "serviceAccountKey"]; + +/// What an obfuscated read shows a caller who may not see the real credential. +const OBJECT_STORE_SECRET_MASK: &str = "*****"; + +/// Blank the secrets in one worker-group config, in place. +/// +/// Worker-group configs are instance-global and expose `env_vars_static` and the bucket +/// credentials of `object_store_cache_config`; a job token (capped at workspace admin) gets this +/// view even when its identity is a superadmin, as does a devops user who is not an instance +/// admin. See `is_instance_admin` (GHSA-hfh4-cx4h-3fcr). Every route that returns a worker-group +/// config must go through here — a single unobfuscated read hands over the whole bucket. +fn obfuscate_worker_config(config: &mut serde_json::Value) { + let Some(config) = config.as_object_mut() else { + return; + }; + if let Some(env_vars) = config + .get_mut("env_vars_static") + .and_then(|v| v.as_object_mut()) + { + for (_, value) in env_vars.iter_mut() { + // the value is a string, so to_string() it and take -2 to drop the quotes + *value = serde_json::json!("*".repeat(value.to_string().len().saturating_sub(2))); + } + } + if let Some(store) = config + .get_mut("object_store_cache_config") + .and_then(|v| v.as_object_mut()) + { + for key in OBJECT_STORE_SECRET_KEYS { + if let Some(secret) = store.get_mut(*key) { + *secret = serde_json::json!(OBJECT_STORE_SECRET_MASK); + } + } + } +} + +/// Put back the credentials behind [`OBJECT_STORE_SECRET_MASK`]. A devops user who is not an +/// instance admin edits the group from the obfuscated view, so a plain save would otherwise +/// store the mask as the secret and take the group's dependency cache offline — silently, since +/// a worker that cannot build its override just falls back to caching on local disk. +async fn restore_masked_object_store_secrets( + db: &DB, + name: &str, + config: &mut serde_json::Value, +) -> error::Result<()> { + if !has_masked_object_store_secret(config) { + return Ok(()); + } + let stored = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name = $1", + name + ) + .fetch_optional(db) + .await? + .map(|c| c.config); + restore_object_store_secrets(config, stored.as_ref()); + Ok(()) +} + +fn has_masked_object_store_secret(config: &serde_json::Value) -> bool { + let Some(store) = config.get("object_store_cache_config") else { + return false; + }; + OBJECT_STORE_SECRET_KEYS + .iter() + .any(|k| store.get(k).and_then(|v| v.as_str()) == Some(OBJECT_STORE_SECRET_MASK)) +} + +/// The half of [`restore_masked_object_store_secrets`] after the read. +fn restore_object_store_secrets( + config: &mut serde_json::Value, + stored: Option<&serde_json::Value>, +) { + let stored = stored + .and_then(|c| c.get("object_store_cache_config")) + .and_then(|v| v.as_object()) + .cloned(); + let Some(store) = config + .get_mut("object_store_cache_config") + .and_then(|v| v.as_object_mut()) + else { + return; + }; + for key in OBJECT_STORE_SECRET_KEYS { + if store.get(*key).and_then(|v| v.as_str()) != Some(OBJECT_STORE_SECRET_MASK) { + continue; + } + match stored.as_ref().and_then(|s| s.get(*key)) { + Some(secret) => store.insert(key.to_string(), secret.clone()), + // Nothing to put back: drop the mask rather than store it. + None => store.remove(*key), + }; + } +} + async fn list_worker_groups( authed: ApiAuthed, Extension(db): Extension, ) -> error::JsonResult> { - let mut configs_raw = - sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name LIKE 'worker__%'") - .fetch_all(&db) - .await?; + let mut configs_raw = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name LIKE 'worker__%'" + ) + .fetch_all(&db) + .await?; // Remove the 'worker__' prefix from all config names for config in configs_raw.iter_mut() { if let Some(name) = &config.name { @@ -75,44 +177,12 @@ async fn list_worker_groups( } } } - // Worker-group configs are instance-global and expose env_vars_static (may hold - // secrets); a job token (capped at workspace admin) gets the obfuscated view even - // when its identity is a superadmin. See is_instance_admin (GHSA-hfh4-cx4h-3fcr). - let configs = if !is_instance_admin(&authed) { - let mut obfuscated_configs: Vec = vec![]; - for config in configs_raw { - let config_value_opt = config.config.as_object().map(|obj| obj.to_owned()); - if let Some(mut config_value) = config_value_opt { - if let Some(env_var_map) = config_value - .get("env_vars_static") - .map(|obj| obj.as_object()) - .flatten() - { - let mut new_env_var_map: serde_json::Map = - serde_json::Map::new(); - for (key, value) in env_var_map { - new_env_var_map.insert( - key.to_owned(), - // we know the value is a string here, so we to_string() it and take -2 to remove the quotes - serde_json::json!("*".repeat(value.to_string().len() - 2)), - ); - } - config_value.insert( - "env_vars_static".to_string(), - serde_json::Value::Object(new_env_var_map), - ); - } - obfuscated_configs.push(Config { - name: config.name, - config: serde_json::Value::Object(config_value), - }) - } + if !is_instance_admin(&authed) { + for config in configs_raw.iter_mut() { + obfuscate_worker_config(&mut config.config); } - obfuscated_configs - } else { - configs_raw - }; - Ok(Json(configs)) + } + Ok(Json(configs_raw)) } async fn get_config( @@ -122,10 +192,20 @@ async fn get_config( ) -> error::JsonResult> { require_devops_role(&db, &authed).await?; - let config = sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name = $1", name) - .fetch_optional(&db) - .await? - .map(|c| c.config); + let mut config = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name = $1", + name + ) + .fetch_optional(&db) + .await? + .map(|c| c.config); + + if !is_instance_admin(&authed) { + if let Some(config) = config.as_mut() { + obfuscate_worker_config(config); + } + } Ok(Json(config)) } @@ -134,10 +214,14 @@ async fn update_config( Path(name): Path, Extension(db): Extension, authed: ApiAuthed, - Json(config): Json, + Json(mut config): Json, ) -> error::Result { require_devops_role(&db, &authed).await?; + if name.starts_with("worker__") { + restore_masked_object_store_secrets(&db, &name, &mut config).await?; + } + #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode @@ -321,9 +405,14 @@ async fn list_configs( Extension(db): Extension, ) -> error::JsonResult> { require_devops_role(&db, &authed).await?; - let configs = sqlx::query_as!(Config, "SELECT name, config FROM config") + let mut configs = sqlx::query_as!(Config, "SELECT name, config FROM config") .fetch_all(&db) .await?; + if !is_instance_admin(&authed) { + for config in configs.iter_mut() { + obfuscate_worker_config(&mut config.config); + } + } Ok(Json(configs)) } @@ -414,3 +503,49 @@ async fn list_all_dedicated_with_deps( Ok(Json(result)) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The mask an obfuscated read hands out must never be storable as the credential itself: + /// a devops user who is not an instance admin edits the group from that view, and a worker + /// that cannot build its override degrades to a local-only cache without failing a job, so + /// the breakage would go unnoticed. + #[test] + fn masked_secrets_survive_a_save_from_the_obfuscated_view() { + let stored = serde_json::json!({ + "object_store_cache_config": { + "type": "S3", "bucket": "cache", "access_key": "AKIA", "secret_key": "s3cr3t" + }, + "env_vars_static": { "TOKEN": "hunter2" }, + }); + + let mut shown = stored.clone(); + obfuscate_worker_config(&mut shown); + let store = &shown["object_store_cache_config"]; + assert_eq!(store["secret_key"], OBJECT_STORE_SECRET_MASK); + assert_eq!(store["access_key"], OBJECT_STORE_SECRET_MASK); + assert_eq!(store["bucket"], "cache"); + assert_ne!(shown["env_vars_static"]["TOKEN"], "hunter2"); + + let mut saved = shown.clone(); + saved["object_store_cache_config"]["bucket"] = serde_json::json!("other"); + restore_object_store_secrets(&mut saved, Some(&stored)); + let store = &saved["object_store_cache_config"]; + assert_eq!(store["secret_key"], "s3cr3t"); + assert_eq!(store["access_key"], "AKIA"); + assert_eq!(store["bucket"], "other"); + } + + #[test] + fn a_mask_with_nothing_behind_it_is_dropped_rather_than_stored() { + let mut saved = serde_json::json!({ + "object_store_cache_config": { "type": "S3", "secret_key": OBJECT_STORE_SECRET_MASK } + }); + restore_object_store_secrets(&mut saved, None); + assert!(saved["object_store_cache_config"] + .get("secret_key") + .is_none()); + } +} diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index 35816dfaa8..1de99fb44f 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -418,7 +418,7 @@ impl EmbeddingsDb { let hub_resource_types = response.json::>().await?; let resource_types: Vec = - sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name",) + sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name",) .fetch_all(pg_db) .await?; diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index ecb7ba405b..525e3cca46 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -77,6 +77,10 @@ pub fn workspaced_service() -> Router { "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}", get(list_paths_from_workspace_runnable), ) + .route( + "/list_paths_linking_agent/{*path}", + get(list_paths_linking_agent), + ) .route("/history_update/v/{version}", post(update_flow_history)) .route("/get/v/{version}", get(get_flow_version_by_id)) .route("/get/v/{version}/p/{*path}", get(get_flow_version)) @@ -508,7 +512,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -521,7 +525,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -534,6 +538,30 @@ async fn list_paths_from_workspace_runnable( Ok(Json(runnables)) } +/// Flows with a step linked to the `ai_agent` resource at `path`, as of their last deploy. +async fn list_paths_linking_agent( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:agent/{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let flows = sqlx::query_scalar!( + r#"SELECT DISTINCT f.path + FROM workspace_runnable_dependencies wru + JOIN flow f + ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id + WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2"#, + path, + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(flows)) +} + async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> { #[cfg(not(feature = "enterprise"))] if new_flow.ws_error_handler_muted.is_some_and(|val| val) { @@ -827,6 +855,29 @@ async fn create_flow( WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() }, ); + // Trigger CI tests for items that reference this flow + { + let db2 = db.clone(); + let w_id2 = w_id.clone(); + let flow_path2 = nf.path.clone(); + let email2 = authed.email.clone(); + let username2 = authed.username.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_dep_map::ci_tests::trigger_ci_tests_for_item( + &db2, + &w_id2, + &flow_path2, + "flow", + &email2, + &username2, + ) + .await + { + tracing::error!(%e, "error triggering CI tests after flow creation"); + } + }); + } + Ok((StatusCode::CREATED, nf.path.to_string())) } @@ -922,7 +973,7 @@ async fn derived_on_behalf_of_email( let Some(permissioned_as) = flow.on_behalf_of.as_deref() else { return Ok(None); }; - // Uncached, for the reason given on `prefetch_cached_script`: this pair is round-tripped. + // Uncached: this pair is round-tripped by the client and stored again on redeploy. Ok(Some( windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) .await?, diff --git a/backend/windmill-api-groups/Cargo.toml b/backend/windmill-api-groups/Cargo.toml index 856eaacc0a..35443008f0 100644 --- a/backend/windmill-api-groups/Cargo.toml +++ b/backend/windmill-api-groups/Cargo.toml @@ -29,4 +29,5 @@ serde.workspace = true serde_json.workspace = true sql-builder.workspace = true sqlx.workspace = true +tracing.workspace = true uuid.workspace = true diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index c6f88dace5..07b8004d83 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -318,6 +318,19 @@ async fn add_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } @@ -528,6 +541,19 @@ async fn remove_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } } diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index a15b9823d3..c931cf2255 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -22,7 +22,10 @@ use windmill_common::{ error::{Error, JsonResult, Result}, utils::{not_found_if_none, paginate, Pagination}, }; -use windmill_common::{db::UserDB, users::username_to_permissioned_as}; +use windmill_common::{ + db::UserDB, + users::{username_to_permissioned_as, usr_accepts_email}, +}; use serde::{Deserialize, Serialize}; use sqlx::{query_scalar, FromRow, Postgres, Transaction}; @@ -972,6 +975,15 @@ async fn add_user_igroup( ) -> Result { require_super_admin(&db, &authed).await?; + // `email_to_igroup` has no shape constraint of its own; `usr`, which the member is + // promoted into on reconcile, has `proper_email`, and a value failing it there would + // roll back every member of the group. + if !usr_accepts_email(&db, &email).await? { + return Err(Error::BadRequest(format!( + "'{email}' is not a valid email address" + ))); + } + let mut tx: Transaction<'_, Postgres> = db.begin().await?; // FOR UPDATE: the group row is the group-level mutex, taken before the workspace @@ -1424,6 +1436,16 @@ async fn overwrite_igroups( if let Some(emails) = &igroup.emails { for email in emails.iter() { + // An export can carry a member the source instance stored before ingest + // validated member values; it is dropped rather than failing the import. + if !usr_accepts_email(&mut *tx, email).await? { + tracing::warn!( + "Skipping member '{}' of imported instance group '{}': not an email address", + email, + igroup.name + ); + continue; + } sqlx::query!( "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", email, diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs new file mode 100644 index 0000000000..60f846ca1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response { + authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap() +} + +/// A `dbt://` relation is one graph node only while every side spells it the same +/// way, and three sides derive that spelling independently: the `// materialize` +/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and +/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer` +/// prove the predicate; only a deploy proves the handler feeds it the key the +/// table actually holds — so a canonicalization that drifted on one side would +/// pass those and split the node here. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + sqlx::query!( + r#"UPDATE workspace_settings + SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb + WHERE workspace_id = 'test-workspace'"# + ) + .execute(&db) + .await?; + + // Nothing generates warehouse DDL, so a managed target is refused rather than + // degraded into the track-only mode it would silently become. + let resp = deploy( + port, + "u/test-user/managed", + "// materialize dbt://main/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("must be `manual`")); + + // The warehouse segment is the identity a dbt model keys on; a name the + // workspace does not configure strands the write on an unreachable node. + let resp = deploy( + port, + "u/test-user/unknown_wh", + "// materialize manual dbt://nope/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("does not configure")); + + // Only the DuckDB executor runs `// data_test` probes, and it runs them around + // a managed write — so a declarer in another language would deploy green with + // its assertions silently never executed. + let resp = deploy( + port, + "u/test-user/tested", + "// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp + .text() + .await? + .contains("`// data_test` is not supported")); + + // Both halves are held to the same relation: every producer is a whole + // `//` under a configured warehouse, so a + // subscription to anything else names something nothing can ever write. + for (path, ref_, expected) in [ + ( + "u/test-user/partial_sub", + "dbt://main/analytics", + "not a whole warehouse relation", + ), + ( + "u/test-user/unknown_wh_sub", + "dbt://nope/analytics/orders", + "does not configure", + ), + ] { + let resp = deploy( + port, + path, + &format!("// on {ref_}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains(expected)); + } + + // Past `asset.path`'s column, where the manifest ingest drops the relation and + // no producer row can exist on either side — computed from the bound so it + // cannot drift under it. + let overlong = format!( + "main/analytics/{}", + "o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN) + ); + let resp = deploy( + port, + "u/test-user/overlong_sub", + &format!("// on dbt://{overlong}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("characters an asset path holds")); + + // Any language may declare the write — the DuckLake write engine is DuckDB's, + // this declaration is not — and the target is canonicalized on the way into + // `asset`, so a hand-written mixed-case spelling lands on the model's key. + let resp = deploy( + port, + "u/test-user/ingest", + "// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + // The create response, not `{:x}` over the stored i64: `ScriptHash` decodes + // hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash + // under 2^60 would 422 the rename below instead of reaching the refusal. + let ingest_hash = resp.text().await?; + let write = sqlx::query_scalar!( + "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \ + AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'" + ) + .fetch_one(&db) + .await?; + assert_eq!(write, "main/analytics/orders"); + + // That producer is native, so subscribing to what it writes is accepted — and + // the `// on` ref has to canonicalize identically, or the row it stores names + // a relation nothing produces. + let resp = deploy( + port, + "u/test-user/consumer", + "// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + let trigger_ref = sqlx::query_scalar!( + "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \ + AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'" + ) + .fetch_one(&db) + .await?; + assert_eq!(trigger_ref, "dbt://main/analytics/orders"); + + // With dbt as the only producer the same subscription can never be woken — a + // dbt run does not dispatch — so the deploy refuses it and names the project. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = deploy( + port, + "u/test-user/mart_consumer", + "// on dbt://main/analytics/MARTS\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // A rename is the other half of that: the producer's write still sits at the + // OLD path in the committed snapshot this deploy reads, while the same + // transaction removes it — so it must not count as the producer that would + // wake the subscription the rename adds. + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/ingest_renamed", + "parent_hash": ingest_hash, + "summary": "", + "description": "", + "content": "// on dbt://main/analytics/orders\nexport async function main() {}", + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index f86522aae3..26e5caeea1 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -913,3 +913,140 @@ async fn test_preserve_orphaned_members_migration(db: Pool) -> anyhow: Ok(()) } + +/// A membership row whose value is not an email (an IdP object id a SCIM sync stored before +/// member values were validated) must not break the workspace's instance-group save: the +/// reconciler skips it and still provisions the valid members. The admin endpoint refuses to +/// add such a value in the first place. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_instance_group_member_that_is_not_an_email_is_skipped( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + const ENTRA_OBJECT_ID: &str = "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed"; + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "entra_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create"); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": "kept@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser"); + + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": ENTRA_OBJECT_ID })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value that is not an email" + ); + let too_wide = format!("{}@example.com", "a".repeat(244)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": too_wide })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value wider than the email columns" + ); + // A valid address whose local part is wider than the username columns: the derived + // username is cut to fit rather than failing the promotion. + let long_local_part = format!("{}@example.com", "a".repeat(60)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": long_local_part })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser long local part"); + + sqlx::query("INSERT INTO email_to_igroup (email, igroup) VALUES ($1, 'entra_grp')") + .bind(ENTRA_OBJECT_ID) + .execute(&db) + .await?; + + // A member whose address only the wider `proper_email` of `usr` accepts, already + // provisioned through the group: reconciliation must keep and re-role them, since + // removal destroys their drafts, inputs and permissions. + sqlx::raw_sql( + r#" + INSERT INTO email_to_igroup (email, igroup) VALUES ('"quoted"@example.com', 'entra_grp'); + INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ('test-workspace', 'quoted', '"quoted"@example.com', false, true, + '{"source": "instance_group", "group": "entra_grp"}'::jsonb); + "#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["entra_grp"], + "roles": { "entra_grp": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let mut members: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + members.sort(); + assert_eq!( + members, + vec![ + ("\"quoted\"@example.com".to_string(), false), + (long_local_part.clone(), false), + ("kept@example.com".to_string(), false), + ], + "valid members provisioned and existing member kept, all as developers; non-email one skipped" + ); + + // A full import carrying the same rows: the object id is dropped, the address only + // `proper_email` accepts is kept, and neither member loses their workspace row. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([{ + "name": "entra_grp", + "emails": ["kept@example.com", "\"quoted\"@example.com", long_local_part, ENTRA_OBJECT_ID] + }])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + let mut stored: Vec = + sqlx::query_scalar("SELECT email FROM email_to_igroup WHERE igroup = 'entra_grp'") + .fetch_all(&db) + .await?; + stored.sort(); + assert_eq!( + stored, + vec![ + "\"quoted\"@example.com".to_string(), + long_local_part.clone(), + "kept@example.com".to_string(), + ], + "import drops the object id and keeps the rest" + ); + let mut after_import: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + after_import.sort(); + assert_eq!(after_import, members, "import must not evict either member"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/login_link.rs b/backend/windmill-api-integration-tests/tests/login_link.rs new file mode 100644 index 0000000000..5c3a231a48 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/login_link.rs @@ -0,0 +1,192 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn login_link_is_single_use_and_same_origin(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + let mint = |token: &'static str, body: serde_json::Value| { + client() + .post(format!("{base}/users/login_links")) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + }; + + // Only a superadmin mints. + let resp = mint("SECRET_TOKEN_2", json!({"email": "test2@windmill.dev"})).await?; + assert_eq!(resp.status(), 401); + + // A superadmin account is never a valid target: the minting credential must not + // become an instance-wide role. + let resp = mint("SECRET_TOKEN", json!({"email": "test@windmill.dev"})).await?; + assert_eq!(resp.status(), 400); + + // An off-origin destination is refused before anything is minted. + let resp = mint( + "SECRET_TOKEN", + json!({"email": "test2@windmill.dev", "rd": "https://evil.example/"}), + ) + .await?; + assert_eq!(resp.status(), 400); + + let resp = mint( + "SECRET_TOKEN", + json!({"email": "test2@windmill.dev", "rd": "/user/workspaces?x=1"}), + ) + .await?; + assert_eq!(resp.status(), 201); + let link = resp.json::().await?; + let path = link["url"] + .as_str() + .unwrap() + .split_once("/api") + .unwrap() + .1 + .to_string(); + let consume_url = format!("{base}{path}"); + + // A promotion inside the link's window is re-checked at open time: no session, + // and the link is not spent while the account is privileged. + sqlx::query("UPDATE password SET super_admin = true WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!( + resp.headers()["location"], + "/user/login_link_expired?reason=invalid" + ); + assert!(resp.headers().get("set-cookie").is_none()); + sqlx::query("UPDATE password SET super_admin = false WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + + // First open: session cookie for the target account, redirected to the stored rd. + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!(resp.headers()["location"], "/user/workspaces?x=1"); + assert_eq!(resp.headers()["referrer-policy"], "no-referrer"); + let cookie = resp + .headers() + .get_all("set-cookie") + .iter() + .map(|c| c.to_str().unwrap().to_string()) + .find(|c| c.starts_with("token=")) + .expect("session cookie"); + assert!(cookie.contains("HttpOnly")); + let session = cookie + .split(';') + .next() + .unwrap() + .trim_start_matches("token=") + .to_string(); + let resp = client() + .get(format!("{base}/users/whoami")) + .header("Authorization", format!("Bearer {session}")) + .send() + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?["email"], + "test2@windmill.dev" + ); + + // Second open: burned, no cookie, bounced to the explanation page. + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!( + resp.headers()["location"], + "/user/login_link_expired?reason=used" + ); + assert!(resp.headers().get("set-cookie").is_none()); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn login_link_mint_can_require_a_login_type(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + let mint = || { + client() + .post(format!("{base}/users/login_links")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({"email": "test2@windmill.dev", "require_login_type": "pending_oauth"})) + .send() + }; + + // A password account is not the account the caller created: no link. + let resp = mint().await?; + assert_eq!(resp.status(), 409); + assert!(resp.text().await?.contains("login_type_mismatch")); + + sqlx::query( + "UPDATE password SET login_type = 'pending_oauth', password_hash = NULL WHERE email = 'test2@windmill.dev'", + ) + .execute(&db) + .await?; + let resp = mint().await?; + assert_eq!(resp.status(), 201); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn cloud_trial_offer_go_refuses_a_job_token(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + + // The answer is a signed-in portal login for the account, so the job-token check + // must come before every other gate: a script holding `$WM_TOKEN` is refused outright, + // where a browser session reaches the next check (off cloud, "no offer"). + let job_id = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user-2', 'u/test-user-2', 'script', 'deno', '{}'::jsonb)", + ) + .bind(job_id) + .execute(&db) + .await?; + let job_token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user-2", + "job", + 600, + "test2@windmill.dev", + &job_id, + None, + None, + ) + .await?; + + let go = |token: String| { + client() + .post(format!("{base}/users/cloud_trial_offer/go")) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let resp = go(job_token).await?; + assert_eq!(resp.status(), 403); + assert!(resp.text().await?.contains("job token")); + + let resp = go("SECRET_TOKEN_2".to_string()).await?; + assert_eq!(resp.status(), 404); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index 06c47cbb14..059b3aa2d2 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -18,15 +18,16 @@ use windmill_common::{ variables::{build_crypt, encrypt}, }; use windmill_native_triggers::{ - classify_read_failure, decrypt_oauth_data, delete_native_trigger, - delete_workspace_integration, get_workspace_integration, + classify_read_failure, decrypt_oauth_data, delete_native_trigger, delete_workspace_integration, + get_workspace_integration, github::GitHub, google::{parse_stop_channel_params, should_renew_channel}, - http_error_status, list_native_triggers, map_external_error, + grant_refused, http_error_status, list_native_triggers, map_external_error, + native_trigger_is_enabled, nextcloud::NextCloud, - grant_refused, require_native_integration_use, store_native_trigger, - store_workspace_integration, External, ExternalReadFailure, HttpRequestError, - NativeTriggerConfig, OAuthConfig, ServiceName, + require_native_integration_use, set_native_trigger_enabled, store_native_trigger, + store_workspace_integration, update_native_trigger, External, ExternalReadFailure, + HttpRequestError, NativeTriggerConfig, OAuthConfig, ServiceName, }; // ============================================================================ @@ -456,6 +457,7 @@ async fn test_delete_integration_full_cascade(db: Pool) -> anyhow::Res &trigger_config, json!({"triggerType": "drive"}), None, + true, ) .await?; @@ -546,6 +548,7 @@ async fn test_cleanup_preserves_triggers(db: Pool) -> anyhow::Result<( &trigger_config, json!({"triggerType": "drive"}), None, + true, ) .await?; @@ -603,6 +606,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< }, json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), None, + true, ) .await?; // An unrelated trigger already sitting on the target path must not be reported as moved. @@ -619,6 +623,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< }, json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), None, + true, ) .await?; @@ -784,7 +789,10 @@ fn test_refresh_failures_blame_only_the_grant_they_refuse() { let ok = Some(StatusCode::OK); assert!(grant_refused(ok, r#"{"error":"bad_refresh_token"}"#)); assert!(grant_refused(ok, r#"{"error":"invalid_grant"}"#)); - assert!(!grant_refused(ok, r#"{"access_token":"t","token_type":"bearer"}"#)); + assert!(!grant_refused( + ok, + r#"{"access_token":"t","token_type":"bearer"}"# + )); } /// A service that is busy or broken has not refused anything, and callers react differently to @@ -826,7 +834,9 @@ fn test_transient_service_failures_are_not_refusals() { body: r#"{"message":"Must have admin rights to Repository."}"#.to_string(), }); assert!( - map_external_error(refused).to_string().contains("admin rights"), + map_external_error(refused) + .to_string() + .contains("admin rights"), "a real 403 keeps its guidance" ); } @@ -850,3 +860,115 @@ fn test_only_service_failures_degrade_the_read() { "a non-provider error must pass through unmapped" ); } + +/// The pause switch a webhook delivery is gated on. A trigger arrives enabled, survives an +/// unrelated edit, and an unknown one reads as enabled so a delivery Windmill cannot place is +/// never silently dropped. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_native_trigger_enabled_toggle(db: Pool) -> anyhow::Result<()> { + insert_test_script(&db, "f/test/handler").await?; + let config = NativeTriggerConfig { + script_path: "f/test/handler".to_string(), + is_flow: false, + webhook_token: "abcdefghij1234567890".to_string(), + }; + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + &config, + json!({"event": "OCA\\Files\\Event\\LoadAdditionalScriptsEvent"}), + None, + true, + ) + .await?; + + assert!( + native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await?, + "a new trigger fires" + ); + + assert!( + set_native_trigger_enabled( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + false + ) + .await? + ); + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await? + ); + + // Saving a configuration must not resume a trigger someone paused. + update_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + &config, + None, + Some("edited"), + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await?, + "an edit leaves the pause in place" + ); + + // A recreate registers a fresh trigger and must be able to come up already paused, in one + // write, rather than being enabled for as long as it takes a second call to arrive. + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-2", + &config, + json!({}), + None, + false, + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-2").await? + ); + + // The conflict branch is a re-registration of a trigger that already exists, so it carries no + // opinion about the pause. + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-2", + &config, + json!({}), + None, + true, + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-2").await?, + "re-registering leaves the pause in place" + ); + + assert!( + !set_native_trigger_enabled( + &db, + "test-workspace", + ServiceName::Nextcloud, + "unknown", + true + ) + .await?, + "nothing to toggle" + ); + assert!( + native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "unknown").await?, + "a trigger Windmill has no row for is not treated as paused" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/offboarding.rs b/backend/windmill-api-integration-tests/tests/offboarding.rs index 4434e6a2b5..7bdeab924b 100644 --- a/backend/windmill-api-integration-tests/tests/offboarding.rs +++ b/backend/windmill-api-integration-tests/tests/offboarding.rs @@ -599,3 +599,59 @@ async fn test_offboard_invalid_target(db: Pool) -> anyhow::Result<()> Ok(()) } + +/// A legacy member named `group-ops` canonicalizes to `g/ops`, the principal the real `ops` group +/// runs as. Offboarding the member must not hand the group's runnables to the replacement. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_offboard_group_prefixed_member_keeps_group_identities( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::raw_sql( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('ops-bot@windmill.dev', 'x', 'password', false, true, 'Ops bot', 'group-ops'); + INSERT INTO usr(workspace_id, email, username, is_admin, role) + VALUES ('test-workspace', 'ops-bot@windmill.dev', 'group-ops', false, 'User'); + INSERT INTO group_(workspace_id, name, summary) VALUES ('test-workspace', 'ops', ''); + INSERT INTO app(workspace_id, path, summary, policy, versions, extra_perms) + VALUES ('test-workspace', 'f/shared/ops_app', '', + '{\"execution_mode\": \"publisher\", \"on_behalf_of\": \"g/ops\", + \"on_behalf_of_email\": \"group-ops@windmill.dev\"}', '{}', '{}');", + ) + .execute(&db) + .await?; + + let preview: serde_json::Value = + authed(client().get(ws_url(port, "offboard_preview/group-ops"))) + .send() + .await? + .json() + .await?; + assert!( + preview["executing_on_behalf"]["apps"].is_null(), + "the group's apps are not the member's to reassign: {preview}" + ); + + let resp = authed(client().post(ws_url(port, "offboard/group-ops"))) + .json(&json!({ + "reassign_to": "u/test-user", + "new_on_behalf_of_user": "test-user", + "delete_user": false + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let principal: Option = sqlx::query_scalar( + "SELECT policy->>'on_behalf_of' FROM app + WHERE workspace_id = 'test-workspace' AND path = 'f/shared/ops_app'", + ) + .fetch_one(&db) + .await?; + assert_eq!(principal.as_deref(), Some("g/ops")); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/protection_rules.rs b/backend/windmill-api-integration-tests/tests/protection_rules.rs index 9469554999..e032acef7b 100644 --- a/backend/windmill-api-integration-tests/tests/protection_rules.rs +++ b/backend/windmill-api-integration-tests/tests/protection_rules.rs @@ -123,6 +123,72 @@ async fn test_protection_rules(db: Pool) -> anyhow::Result<()> { .await?; assert!(!resp.status().is_success(), "Non-admin should be blocked from flows: {}", resp.status()); + // ======================================== + // 4b. ...but a draft-only resource stays deletable: nothing is deployed at + // its path, so its DELETE is a draft discard rather than a deployment. + // ======================================== + + let draft_only_path = "u/test-user-2/draft_only_resource"; + let resp = authed( + client().post(format!("{base}/drafts/update/resource/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ "value": { + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + }})) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Non-admin should save a draft: {}", + resp.text().await? + ); + + let resp = authed( + client().delete(format!("{base}/resources/delete/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Draft-only delete should not be gated by the deploy rules: {}", + resp.text().await? + ); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'resource'::DRAFT_KIND", + ) + .bind(draft_only_path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + // The gate itself is still there for a DEPLOYED resource at the same path. + let resp = authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "Non-admin should still be blocked from creating a resource: {}", + resp.status() + ); + // ======================================== // 5. Admin bypasses protection rule // ======================================== diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 3f0b21b184..53648ec759 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -443,6 +443,30 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { let body = resp.json::().await?; assert_eq!(body["description"], "Updated type desc"); + // display_name: an update that omits it, as a push from a CLI predating the field does, + // keeps it; an explicit null clears it. + for (update, expected) in [ + ( + json!({"display_name": "New Test Type"}), + json!("New Test Type"), + ), + ( + json!({"description": "Updated type desc"}), + json!("New Test Type"), + ), + (json!({"display_name": null}), serde_json::Value::Null), + ] { + let resp = authed(client().post(resource_url(port, "type/update", "new_test_type"))) + .json(&update) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let resp = authed_get(port, "type/get", "new_test_type").await; + let body = resp.json::().await?; + assert_eq!(body["display_name"], expected); + } + // type/delete let resp = authed(client().delete(resource_url(port, "type/delete", "new_test_type"))) .send() diff --git a/backend/windmill-api-integration-tests/tests/schedules.rs b/backend/windmill-api-integration-tests/tests/schedules.rs index 89ba0ec2cc..881f11ff43 100644 --- a/backend/windmill-api-integration-tests/tests/schedules.rs +++ b/backend/windmill-api-integration-tests/tests/schedules.rs @@ -113,7 +113,9 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { "expected at least 2 schedules, got {}", list.len() ); - assert!(list.iter().any(|s| s["path"] == "u/test-user/test_schedule")); + assert!(list + .iter() + .any(|s| s["path"] == "u/test-user/test_schedule")); // --- list_with_jobs --- let resp = authed(client().get(format!("{base}/list_with_jobs"))) @@ -125,18 +127,14 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert!(!list.is_empty()); // --- update --- - let resp = authed(client().post(schedule_url( - port, - "update", - "u/test-user/test_schedule", - ))) - .json(&json!({ - "schedule": "0 0 */12 * * *", - "timezone": "Europe/Paris" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(schedule_url(port, "update", "u/test-user/test_schedule"))) + .json(&json!({ + "schedule": "0 0 */12 * * *", + "timezone": "Europe/Paris" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); // verify update @@ -204,14 +202,11 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(resp.status(), 200); // --- delete --- - let resp = authed(client().delete(schedule_url( - port, - "delete", - "u/test-user/another_schedule", - ))) - .send() - .await - .unwrap(); + let resp = + authed(client().delete(schedule_url(port, "delete", "u/test-user/another_schedule"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let resp = authed_get(port, "exists", "u/test-user/another_schedule").await; @@ -220,17 +215,108 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Global endpoints ===== // --- preview --- - let resp = authed(client().post(format!( - "http://localhost:{port}/api/schedules/preview" - ))) - .json(&json!({ - "schedule": "0 0 */6 * * *", - "timezone": "UTC" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(format!("http://localhost:{port}/api/schedules/preview"))) + .json(&json!({ + "schedule": "0 0 */6 * * *", + "timezone": "UTC" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "preview: {}", resp.text().await?); Ok(()) } + +/// A schedule with no `schedule` row is listed from the `draft` table, so its +/// DELETE drops that draft, then 404s once nothing is left at the path. A legacy +/// (`email IS NULL`) draft is owned by nobody and stays put. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_draft_only_schedule(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/draft_only_schedule"; + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/trigger_schedule/{path}" + ))) + .json(&json!({ "value": { + "path": path, + "schedule": "0 0 */6 * * *", + "timezone": "UTC", + "script_path": "u/test-user/never_deployed", + "is_flow": false, + }})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "save draft: {}", resp.text().await?); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/schedules/list?include_draft_only=true" + ))) + .send() + .await + .unwrap(); + let listed: Vec = resp.json().await?; + assert!( + listed + .iter() + .any(|s| s["path"] == path && s["draft_only"] == json!(true)), + "draft-only schedule should be listed: {listed:?}" + ); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "nothing left at the path"); + + // A legacy (email IS NULL) draft is owned by nobody and keeps the write gate + // on the drafts routes, so this one must not become a second door to it. + let legacy_path = "u/test-user/legacy_draft_only_schedule"; + sqlx::query( + "INSERT INTO draft (workspace_id, email, path, typ, value) \ + VALUES ('test-workspace', NULL, $1, 'trigger_schedule'::DRAFT_KIND, '{}'::json)", + ) + .bind(legacy_path) + .execute(&db) + .await?; + + let resp = authed(client().delete(schedule_url(port, "delete", legacy_path))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 404, + "legacy draft is not this route's to delete" + ); + + let legacy_remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(legacy_path) + .fetch_one(&db) + .await?; + assert_eq!(legacy_remaining, 1, "the legacy draft should survive"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs index 4e25be1fa5..dc8e8dcc00 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -923,6 +923,7 @@ async fn test_pull_stays_on_the_workspace_lane(db: Pool) -> anyhow::Re &db, "test-workspace", &repo, + ("test-user", "test@windmill.dev"), None, false, None, diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index e897bf5eaf..310e44ae1f 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -38,8 +38,8 @@ use windmill_common::{ FlowVersionInfo, DB, }; use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, push, PushArgs, PushArgsOwned, - PushIsolationLevel, + cancel_job, get_result_and_success_by_id_from_flow, parse_result_object, push, PushArgs, + PushArgsOwned, PushIsolationLevel, }; use crate::types::RunJobQuery; @@ -374,9 +374,9 @@ pub async fn run_wait_result_internal( } pub fn result_to_response(result: Box, success: bool) -> error::Result { - let composite_result = serde_json::from_str::(result.get()); + let composite_result = parse_result_object::(result.get()); match composite_result { - Ok(WindmillCompositeResult { + Some(WindmillCompositeResult { windmill_status_code, windmill_content_type, windmill_headers, @@ -1192,4 +1192,13 @@ mod result_to_response_tests { assert!(res.is_err(), "hop-by-hop header must be rejected: {name}"); } } + + #[tokio::test] + async fn array_result_is_not_a_composite_response() { + let json = r#"[201,"text/html",null,null,"

hi

"]"#; + let resp = result_to_response(raw(json), true).expect("response"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(body_bytes(resp).await, json.as_bytes()); + } } diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index 835c4c7e34..857b2e02e2 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -585,6 +585,8 @@ pub fn list_completed_jobs_query( let mut sqlb = SqlBuilder::select_from("v2_job_completed") .fields(fields) .order_by( + // The runs page picks its pagination cursor column from this same rule + // (frontend/src/lib/components/runs/useJobsLoader.svelte.ts); change both together. if lq.completed_before.is_some() || lq.completed_after.is_some() || lq.success == Some(false) diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 6ae902e984..dfd4a6e3ba 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -31,8 +31,8 @@ use windmill_common::{ self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, }, user_drafts::{ - delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, - UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, fetch_draft_only_list_rows, + overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, }, utils::{ escape_ilike_pattern, not_found_if_none, paginate, Pagination, ScheduleType, StripPath, @@ -332,7 +332,7 @@ async fn create_schedule( ) .await?; // email is still written for backwards compat with old workers that don't know about permissioned_as - let resolved_email = windmill_common::users::get_email_from_permissioned_as( + let resolved_email = windmill_common::users::get_email_from_permissioned_as_uncached( &resolved_permissioned_as, &w_id, &db, @@ -545,18 +545,14 @@ async fn edit_schedule( reject_reserved_schedule_path(path)?; let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; - let mut tx = user_db.begin(&authed).await?; // Check schedule for error ScheduleType::from_str(&es.schedule, es.cron_version.as_deref(), true)?; - // Validate dynamic_skip if provided - if let Some(handler_path) = &es.dynamic_skip { - validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; - } - let resolved_edited_by = resolve_edited_by(&authed); + // Resolved on the (non-RLS) pool before the RLS transaction opens: the lookup mid-transaction + // would hold a second connection while `tx` is checked out. let resolved_permissioned_as = resolve_permissioned_as( es.permissioned_as.as_ref(), es.preserve_permissioned_as, @@ -568,7 +564,7 @@ async fn edit_schedule( let resolved_email = if resolved_permissioned_as != windmill_common::users::username_to_permissioned_as(&authed.username) { - windmill_common::users::get_email_from_permissioned_as( + windmill_common::users::get_email_from_permissioned_as_uncached( &resolved_permissioned_as, &w_id, &db, @@ -585,6 +581,13 @@ async fn edit_schedule( Some(&resolved_email), )?; + let mut tx = user_db.begin(&authed).await?; + + // Validate dynamic_skip if provided + if let Some(handler_path) = &es.dynamic_skip { + validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; + } + let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; let schedule = sqlx::query_as!( @@ -1126,35 +1129,23 @@ pub async fn set_enabled( check_scopes(&authed, || format!("schedules:write:{}", path))?; reject_reserved_schedule_path(path)?; - // Block enabling a schedule in a fork when the parent has the same path - // (regardless of parent's enabled flag), unless force=true. Two enabled - // crons fire in lockstep; even when the parent is currently disabled the - // user is likely to re-enable it later, at which point both fire — better - // to surface that risk at every fork-side enable. There's no namespacing - // fix for schedules (Phase 3 doesn't help cron); the user has to confirm - // or point the script at fork-only side effects. + // Block enabling a schedule in a fork when an ancestor has the same path + // (regardless of its enabled flag), unless force=true. Two enabled crons + // fire in lockstep; even when the ancestor is currently disabled the user + // is likely to re-enable it later, at which point both fire — better to + // surface that risk at every fork-side enable. There's no namespacing fix + // for schedules (Phase 3 doesn't help cron); the user has to confirm or + // point the script at fork-only side effects. if payload.enabled && !payload.force { - let parent_id: Option = sqlx::query_scalar!( - "SELECT parent_workspace_id FROM workspace WHERE id = $1", - &w_id + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, "schedule", &w_id, path, ) - .fetch_optional(&mut *tx) .await? - .flatten(); - if let Some(parent_id) = parent_id { - let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM schedule WHERE workspace_id = $1 AND path = $2)", - &parent_id, - path, - ) - .fetch_one(&mut *tx) - .await?; - if exists == Some(true) { - return Err(Error::BadRequest(format!( - "fork-conflict:schedule:{}", - parent_id - ))); - } + { + return Err(Error::BadRequest(format!( + "fork-conflict:schedule:{}", + ancestor_id + ))); } } let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; @@ -1331,6 +1322,18 @@ async fn delete_schedule( .flatten(); if exists.is_none() { + drop(tx); + if delete_draft_only_for_path( + &db, + &w_id, + UserDraftItemKind::TriggerSchedule, + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only schedule {} deleted", path)); + } return Err(windmill_common::error::Error::NotFound(format!( "Schedule {} not found", path @@ -1684,9 +1687,9 @@ pub use windmill_queue::schedule::clear_schedule; #[derive(Deserialize)] pub struct SetEnabled { pub enabled: bool, - /// Bypass the parent-state warning when enabling a schedule in a fork - /// whose parent has the same path enabled. The frontend sets this after - /// the user confirms the duplicate-firing dialog. + /// Bypass the fork-conflict warning when enabling a schedule in a fork + /// while an ancestor workspace has the same path. The frontend sets this + /// after the user confirms the duplicate-firing dialog. #[serde(default)] pub force: bool, } diff --git a/backend/windmill-api-scripts/src/asset_inference.rs b/backend/windmill-api-scripts/src/asset_inference.rs index 9c503ae1ae..1c0bc6d57f 100644 --- a/backend/windmill-api-scripts/src/asset_inference.rs +++ b/backend/windmill-api-scripts/src/asset_inference.rs @@ -78,20 +78,26 @@ fn comment_prefix(lang: &ScriptLang) -> Option<&'static str> { | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Nativets - | ScriptLang::Go => Some("//"), + | ScriptLang::Go + | ScriptLang::Php => Some("//"), _ => None, } } /// Mirror of the frontend `parseVolumeAnnotations` (infer.ts): ` /// volume: ` lines in the leading comment block, each an `rw` volume -/// asset. Scanning stops at the first non-comment line (blank lines are -/// skipped), exactly like the frontend. +/// asset. Scanning stops at the first non-comment line; blank lines and PHP's +/// opening tag line are skipped whole (the tag may carry code, so an annotation +/// must sit on its own line below it), exactly like the frontend. fn parse_volume_annotations(content: &str, prefix: &str) -> Vec { let mut volumes = Vec::new(); for line in content.lines() { let trimmed = line.trim(); - if trimmed.is_empty() { + if trimmed.is_empty() + || trimmed + .get(..5) + .is_some_and(|p| p.eq_ignore_ascii_case(" = got.iter().filter(|a| a.kind == AssetKind::Volume).collect(); + assert_eq!(vols.len(), 1); + assert_eq!(vols[0].path, "my_vol"); + } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index ccc5815aa7..a4667e4876 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,8 +39,8 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_common::{ assets::{ @@ -1081,6 +1081,57 @@ fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { [(path.to_string(), hash_script(lock))] } +/// The `dbt://` relation both halves of a deploy have to agree on: a whole +/// `//`, under a warehouse this workspace configures. +/// +/// Every producer is held to exactly this — a `// materialize` target here, a +/// descriptor's `profile.warehouse` in the worker — so a subscription to anything +/// else names a relation nothing can ever write. No later deploy fixes that and +/// no dormant-edge warning reports it, since the warning fires on a dbt project's +/// ingest and no project can claim a relation under a warehouse that isn't there. +/// Asking here rather than at each site is what keeps the two from drifting into +/// refusing and accepting the same string. +async fn validate_dbt_relation( + db: &sqlx::Pool, + w_id: &str, + relation: &str, + what: &str, +) -> Result<()> { + if !windmill_parser::asset_parser::is_full_relation_path(relation) { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is not a whole warehouse relation \ + (`dbt:////`)." + ))); + } + // `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that + // outgrows it rather than failing the whole graph, so past the column no + // producer row can exist on either side — a write would be rejected by + // Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text + // that would take the subscription and keep it dormant for good. + let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN; + if relation.chars().count() > max { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is longer than the {max} characters an asset path \ + holds, so it cannot be recorded." + ))); + } + let warehouse = relation.split('/').next().unwrap_or_default(); + // Only the resolver's own "no such warehouse" is the annotation's fault. Its + // other failures — the query, and a setting entry with no `resource_path` — + // keep their own error: blaming the warehouse name for those misdescribes + // them, and flattening the malformed-setting one to a 400 hides a server + // fault behind a client one. + windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse) + .await + .map_err(|e| match e { + Error::NotFound(_) => Error::BadRequest(format!( + "{what} `dbt://{relation}` names a warehouse this workspace does not \ + configure: {e}" + )), + other => other, + }) +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1267,6 +1318,47 @@ async fn create_script_internal<'c>( } } + // A retired path keeps its versions, and the newest is where a redeploy belongs: hashed + // as a first deploy instead, unchanged content lands on the row the path's own first + // version already holds. A deleted version still counts — its row keeps the hash it was + // deployed under even once the content is wiped, so skipping it is what collides. + // + // Any parentless deploy, not only an `auto_parent` one: the CLI names no parent for a + // path its listing no longer shows, which is where a retried push lands. Gated on + // nothing being live there, so a parentless deploy onto a live path still meets the + // path conflict the match below raises. + let mut parent_adopted_from_retired_path = false; + if ns.parent_hash.is_none() && clashing_script.is_none() { + // Locked, not merely read: a competing deploy chaining onto this same candidate + // takes `FOR UPDATE` on it before inserting, so holding the row is what serializes + // the two. Probe first and the child still uncommitted reads as absent. + let candidate = sqlx::query_scalar::<_, i64>( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 \ + ORDER BY created_at DESC LIMIT 1 FOR UPDATE", + ) + .bind(&ns.path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + // Adoptable only if nothing already descends from it: a rename leaves its source + // path holding a version whose child lives at the destination, and a second child + // forks a lineage the guard below requires to be linear. Nothing adoptable means a + // fresh lineage, which has no parent to vary its hash and can still collide. + ns.parent_hash = match candidate { + Some(hash) => sqlx::query_scalar!( + "SELECT 1 FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", + hash, + &w_id + ) + .fetch_optional(&db) + .await? + .is_none() + .then_some(ScriptHash(hash)), + None => None, + }; + parent_adopted_from_retired_path = ns.parent_hash.is_some(); + } + // Must stay below the parent resolution above: an auto_parent deploy hashed before // it carries a first deploy's lineage, so redeploying content the path has held // before collides with that archived version instead of superseding it. The @@ -1311,20 +1403,42 @@ async fn create_script_internal<'c>( )); }; + // Unscoped, and sound only under the lock above: linearity is a property of the + // lineage, not of what this caller may read. A child can sit where they cannot + // see it — a folder they renamed it into, or grants an adopting deploy reset — + // and asked through `tx` it reads as absent, letting the fork through. let clashing_hash_o = sqlx::query_scalar!( "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", p_hash.0, &w_id ) - .fetch_optional(&mut *tx) + .fetch_optional(&db) .await?; if let Some(clashing_hash) = clashing_hash_o { - return Err(Error::BadRequest(format!( - "A script with hash {} with same parent_hash has been found. However, the \ - lineage must be linear: no 2 scripts can have the same parent", - ScriptHash(clashing_hash) - ))); + // Named only when the caller could already read it. The probe above has to be + // unscoped to be correct, but a hash alone reads a script's content back + // through `raw/h/{hash}`, which authorizes nothing per script — so echoing one + // the caller cannot see hands them a way to fetch it. + let visible_to_caller = sqlx::query_scalar!( + "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", + clashing_hash, + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + return Err(Error::BadRequest(if visible_to_caller { + format!( + "A script with hash {} with same parent_hash has been found. However, \ + the lineage must be linear: no 2 scripts can have the same parent", + ScriptHash(clashing_hash) + ) + } else { + "A script with the same parent_hash has been found. However, the lineage \ + must be linear: no 2 scripts can have the same parent" + .to_owned() + })); }; let ScriptWithStarred { script: ps, .. } = @@ -1387,7 +1501,15 @@ async fn create_script_internal<'c>( } Some(_) | None => Ok(Some(ParentInfo { p_hashes: ph, - perms: ps.extra_perms, + // A version adopted above was taken for its lineage, not its grants: a + // retired path may be reused by a different script, which must not start + // life holding an ACL nobody gave it — including one `delete/h` purged. + // A parent the caller named still carries them, as unarchive expects. + perms: if parent_adopted_from_retired_path { + json!({}) + } else { + ps.extra_perms + }, p_path: ps.path, })), }; @@ -1565,34 +1687,90 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// materialize` materializes a `ducklake:///` target from a - // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake - // target would otherwise deploy, register a producer in the asset graph, then - // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), - // and a non-DuckDB script never reaches the executor that records state. The - // managed-only checks (single trailing SELECT, no SQL args) come after — a - // `manual` script owns its DDL and skips them. + // `// materialize` names what this script produces. Two target kinds, and the + // runtime behind each is what constrains the annotation: + // • `ducklake:///
` — the DuckDB executor generates the write + // (or, in `manual` mode, records the state the script wrote itself), so + // the script has to be a DuckDB one and the target has to name a table. + // A non-DuckDB script never reaches that executor. + // • `dbt:////` — a warehouse relation. Nothing + // generates warehouse DDL, so the declaration is track-only (`manual`) + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. + // Any other kind would deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). + // The managed-only checks (single trailing SELECT, no SQL args) come after — + // a `manual` script owns its DDL and skips them. if let Some(m) = pipeline_annotations.materialize.as_ref() { - if ns.language != ScriptLang::DuckDb { - return Err(Error::BadRequest(format!( - "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ - wmll.ducklake helpers to materialize from other languages.", - ns.language.as_str() - ))); - } - if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + use windmill_parser::asset_parser::AssetKind as PAssetKind; + // The producer half of the rule the trigger loop below applies to `// on`: + // a dbt project's writes come from its manifest, and the graph ingest + // republishes this path's asset rows wholesale, so a declared one would be + // wiped by the very deploy that accepted it while its runs kept stamping + // the relation. + if ns.language == ScriptLang::Dbt { return Err(Error::BadRequest( - "`// materialize` only supports a DuckLake target \ - (`ducklake:///
`); other asset kinds aren't materializable." + "a dbt script cannot declare `// materialize`: what a project builds is read \ + from its manifest and published by the graph ingest, not annotated." .to_string(), )); } - if !m.target_path.contains('/') { - return Err(Error::BadRequest(format!( - "`// materialize` needs a table in the target: \ - `ducklake://{0}/
` (got `ducklake://{0}`).", - m.target_path - ))); + match m.target_kind { + PAssetKind::Ducklake => { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages, or declare a \ + warehouse relation with `// materialize manual dbt://…`.", + ns.language.as_str() + ))); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + } + PAssetKind::Dbt => { + // `// data_test` runs as verifier probes the DuckDB executor + // splices around a MANAGED write. Nothing generates a warehouse + // write, so nothing would run them — and unlike the DuckLake + // `manual` case, which at least fails loudly in that executor, a + // declarer in another language would deploy green with its + // data-quality assertions silently never executed. + if !pipeline_annotations.data_tests.is_empty() { + return Err(Error::BadRequest( + "`// data_test` is not supported with a `dbt://` target: the checks run \ + against a managed materialization, and a warehouse relation is \ + written by the script itself. Assert on the relation with a dbt \ + test in the project that reads it." + .to_string(), + )); + } + if !m.manual { + return Err(Error::BadRequest( + "`// materialize dbt://…` must be `manual`: Windmill generates no \ + warehouse DDL, so the script issues its own write and only the outcome \ + is recorded. Write \ + `// materialize manual dbt:////`." + .to_string(), + )); + } + validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target") + .await?; + } + _ => { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake (`ducklake:///
`) or \ + warehouse-relation (`dbt:////`) target; other asset \ + kinds aren't materializable." + .to_string(), + )); + } } if !m.manual { if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { @@ -2347,27 +2525,28 @@ async fn create_script_internal<'c>( // while its own finished runs still render from them. Clearing by path // would empty those run pages for good. if ns.language != ScriptLang::Dbt { - // The saved retry state does go: nothing regenerates it, it is keyed by - // path alone, and it carries one user's failed invocation and its - // arguments. No dbt version is live at this path any more to resume it. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, &ns.path).await?; + // The saved run and environment state do go: nothing regenerates them, + // both are keyed by path alone, and they carry one user's failed + // invocation with its arguments and the project's own manifest. No dbt + // version is live at this path any more to resume or defer to. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, &ns.path).await?; } if let Some(ref old) = p_path_opt { if old != &ns.path { clear_script_triggers(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; clear_static_asset_usage(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; - // The saved retry state travels rather than being cleared: nothing + // The saved state travels rather than being cleared: nothing // regenerates it, so dropping it would throw away a resumable - // failure for what is only a rename. Only while the destination is - // still dbt — a rename that also converts the language would - // otherwise reinstate at the new path the state the branch above - // just cleared, leaving one user's arguments and results under a - // path no dbt script occupies. + // failure and every deferral until the next full run, for what is + // only a rename. Only while the destination is still dbt — a rename + // that also converts the language would otherwise reinstate at the + // new path the state the branch above just cleared, leaving one + // user's arguments and results under a path no dbt script occupies. if ns.language == ScriptLang::Dbt { - windmill_common::dbt_manifest::move_dbt_run_state(&mut tx, &w_id, old, &ns.path) + windmill_common::dbt_manifest::move_dbt_script_state(&mut tx, &w_id, old, &ns.path) .await?; } else { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, old).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, old).await?; } } } @@ -2375,16 +2554,43 @@ async fn create_script_internal<'c>( let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { continue; }; - // A `dbt://` subscription can never fire: dbt is the only producer of a - // warehouse relation (`// materialize` takes DuckLake targets only) and a - // dbt run does not dispatch. Refusing beats persisting a row that draws a - // cascade arrow on the canvas and then never wakes anything. - if trigger_ref.starts_with("dbt://") { - return Err(Error::BadRequest(format!( - "`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \ - runs, and nothing else writes a warehouse relation. Declare the read without \ - `on` to keep the lineage edge, or schedule this script." - ))); + // A `dbt://` subscription fires only when a NON-dbt job materialized the + // relation: `// materialize manual dbt://…` declares such a write, while a + // dbt run records its models and does not dispatch. So refuse exactly the + // edge that cannot fire — one whose relation is already claimed by dbt and + // by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`, + // which takes the workspace pool: under RLS an unreadable native producer + // would refuse a live subscription). + if let Some(relation) = trigger_ref.strip_prefix("dbt://") { + // The subscriber side of the same rule: a dbt project is not woken by + // the asset cascade. Its graph ingest clears these rows for its own + // path, so accepting one here would deploy an edge the dependency job + // then silently removes. + if ns.language == ScriptLang::Dbt { + return Err(Error::BadRequest(format!( + "a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \ + and a project is run on its schedule, not woken by an asset cascade." + ))); + } + validate_dbt_relation(&db, &w_id, relation, "subscription target").await?; + // Both paths under a rename: the old one's committed write row is + // still there and this transaction is about to remove it. + let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) { + Some(old) => vec![ns.path.clone(), old.to_string()], + None => vec![ns.path.clone()], + }; + if let Some(dbt_owner) = + windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths) + .await? + { + return Err(Error::BadRequest(format!( + "`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \ + `{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \ + read without `on` to keep the lineage edge, or schedule this script. A \ + relation written by a `// materialize manual {trigger_ref}` script can be \ + subscribed to." + ))); + } } // Effective debounce for this edge: per-`// on debounce=` wins, // else the script-level `// debounce` default. Debounce only @@ -3578,7 +3784,11 @@ async fn archive_script_by_path( path, &w_id ) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as the by-hash routes are: + // committed on its own, a cleanup that then fails leaves dbt state at a path + // no live version occupies, for whatever is created there next to defer + // through. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; @@ -3586,9 +3796,10 @@ async fn archive_script_by_path( // The graph stays: the pinned read resolves versions through a CTE that // already skips archived rows, so it stops answering for current relations // either way, while deleting it would empty the Models panel of every - // completed run of the project. Retry state does go — nothing may resume a - // script that is no longer live. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + // completed run of the project. The saved run and environment state do go — + // nothing may resume a script that is no longer live, and nothing may defer + // through what it last built. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; // Pipeline event hygiene: an archived script must not be triggered by // anything. Wipe declared `// on ...` edges (asset-event subscribers // look these up). @@ -3673,7 +3884,7 @@ async fn archive_script_by_hash( clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; // The version's graph stays: its finished runs still render from it, and // the live-version CTE already skips archived rows. Deletion clears it. - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3736,7 +3947,12 @@ async fn delete_script_by_hash( ) .bind(&hash.0) .bind(&w_id) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as `archive_script_by_hash` + // already does. Committed on its own, it opens a window where the path has + // no live version and a concurrent deploy can take it — and the retirement + // guard below then finds that new script live, keeps the old project's dbt + // state, and leaves the replacement able to defer through its manifest. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; @@ -3749,7 +3965,7 @@ async fn delete_script_by_hash( windmill_common::dbt_manifest::clear_dbt_manifest_version(&mut tx, &w_id, &script.path, hash.0) .await?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3850,11 +4066,11 @@ async fn delete_script_by_path( // After the DELETE, never before: every dbt writer locks the `script` row // first, so taking a sidecar ahead of it deadlocks one of the pair. The - // VERSIONED graph needs no clear at all, cascading off `script`; the retry - // state does, being keyed by path alone and so inherited by whatever is - // created here next, and so do the editor's own graphs, whose NULL + // VERSIONED graph needs no clear at all, cascading off `script`; the saved + // run and environment state do, being keyed by path alone and so inherited + // by whatever is created here next, and so do the editor's own graphs, whose NULL // `script_hash` satisfies that foreign key without riding its cascade. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, path).await?; if !trash_scripts.is_empty() { @@ -4023,7 +4239,7 @@ async fn delete_scripts_bulk( // Same reason as the single-path delete, over every requested path rather // than the deleted ones: a path that had no script left can still hold state. for p in &request.paths { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, p).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, p).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, p).await?; } diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 18a8d52ac0..2e2b3290fa 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -58,9 +58,10 @@ use windmill_common::{ AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, - HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, - RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING, + GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, + RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING, UNIQUE_ID_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WS_BASE_URL_SETTING, @@ -113,6 +114,9 @@ async fn get_ruff_config_unauthed(Extension(db): Extension) -> error::Result pub fn global_service() -> Router { #[warn(unused_mut)] let r = Router::new() + // `/local` is the path in openapi.yaml, so every generated client (getLocal) calls it; + // `/envs` stays for callers that found the route in the code. + .route("/local", get(get_local_settings)) .route("/envs", get(get_local_settings)) .route( "/global/{key}", @@ -1044,6 +1048,12 @@ async fn run_setting_pre_write_hook( } } } + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING => { + // Rejected at write time rather than at boot: a mistyped origin + // matches no request, so it would silently block the very app it + // names with nothing but a log line to go on. + windmill_common::global_settings::parse_allowed_origins_setting(Some(value))?; + } HTTP_ROUTE_WORKSPACED_ROUTE_SETTING => { let serde_json::Value::Bool(workspaced_route) = value else { return Err(error::Error::BadRequest(format!( @@ -1172,6 +1182,18 @@ async fn run_setting_pre_write_hook( } } } + INSTANCE_BANNER_SETTING => { + match value { + // Clearing (delete row) is handled by the caller; allow it through. + serde_json::Value::Null => {} + serde_json::Value::String(s) if s.trim().is_empty() => {} + v => { + windmill_common::global_settings::validate_instance_banner(v).map_err(|e| { + error::Error::BadRequest(format!("{INSTANCE_BANNER_SETTING}: {e}")) + })?; + } + } + } _ => {} } Ok(()) @@ -1306,12 +1328,21 @@ pub async fn get_global_setting( && key != AUTOMATE_USERNAME_CREATION_SETTING && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING + // `wmill hub pull` reads it from a job, and no job token clears the gate. It binds an + // offline license only together with `license_key`, which stays gated. + && key != UNIQUE_ID_SETTING && key != HUB_ACCESSIBLE_URL_SETTING && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING && key != HTTP_ROUTE_WORKSPACED_ROUTE_SETTING + // The route editor shows the inherited default to whoever is editing a + // trigger, who is usually not a superadmin. Not a secret either: any + // browser discovers the list by reading Access-Control-Allow-Origin off + // a response. + && key != HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING && key != WS_BASE_URL_SETTING + && key != INSTANCE_BANNER_SETTING { require_super_admin(&db, &authed).await?; } @@ -2043,6 +2074,13 @@ struct CachedResourceType { deserialize_with = "windmill_common::more_serde::double_option" )] format_extension: Option>, + /// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit + /// null (the hub naming nothing) clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } #[derive(serde::Deserialize)] @@ -2054,6 +2092,11 @@ struct HubResourceTypeRaw { description: Option, #[serde(default)] format_extension: Option, + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } async fn fetch_resource_types_from_hub() -> error::Result> { @@ -2096,6 +2139,7 @@ async fn fetch_resource_types_from_hub() -> error::Result 100 => None, + other => other.clone(), + }; let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), rt.format_extension.clone().flatten(), rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .fetch_one(&db) .await?; @@ -2167,8 +2219,8 @@ async fn sync_cached_resource_types( // Whether the payload carried the key at all is what decides: present // (even as null) is authoritative and may clear, absent means a cache // written before the column and must leave the stored value alone. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) - VALUES ('admins', $1, $2, $3, $4, now()) + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $6, now()) ON CONFLICT (workspace_id, name) DO UPDATE SET schema = EXCLUDED.schema, description = EXCLUDED.description, -- A fileset is a set of files, so it cannot also be one file. @@ -2179,12 +2231,15 @@ async fn sync_cached_resource_types( WHEN resource_type.is_fileset THEN NULL WHEN $5 THEN EXCLUDED.format_extension ELSE resource_type.format_extension END, + display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END, edited_at = now()", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), rt.format_extension.clone().flatten(), rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .execute(&db) .await?; diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index afe7580055..476a900ef6 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -46,15 +46,15 @@ use tracing::Instrument; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::audit::AuditAuthor; -use windmill_common::auth::{safe_token_prefix, TOKEN_PREFIX_LEN}; +use windmill_common::auth::{hash_token, safe_token_prefix, TOKEN_PREFIX_LEN}; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; use windmill_common::oauth2::InstanceEvent; use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; use windmill_common::users::{ - username_to_permissioned_as, PERMISSIONED_AS_MAX_LEN, SUPERADMIN_NOTIFICATION_EMAIL, - SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, + username_to_permissioned_as, EMAIL_COLUMN_MAX_LEN, PERMISSIONED_AS_MAX_LEN, + SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, }; use windmill_common::utils::paginate; use windmill_common::worker::CLOUD_HOSTED; @@ -140,6 +140,16 @@ pub fn global_service() -> Router { ) .route("/tokens/list", get(list_tokens)) .route("/tokens/impersonate", post(impersonate)) + .route("/login_links", post(create_login_link)) + .route( + "/cloud_trial_offer", + post(set_cloud_trial_offer).get(get_cloud_trial_offer), + ) + .route("/cloud_trial_offer/go", post(go_cloud_trial_offer)) + .route( + "/onboarding_profile", + post(set_onboarding_profile).get(get_onboarding_profile), + ) .route("/usage", get(get_usage)) .route("/all_runnables", get(get_all_runnables)) .route("/refresh_token", get(refresh_token)) @@ -158,6 +168,7 @@ pub fn make_unauthed_service() -> Router { .route("/logout", post(logout).get(logout)) .route("/is_first_time_setup", get(is_first_time_setup)) .route("/request_password_reset", post(request_password_reset)) + .route("/login_link/{token}", get(consume_login_link)) .route("/is_smtp_configured", get(is_smtp_configured)) .route( "/is_password_login_disabled", @@ -255,11 +266,14 @@ pub struct WorkspaceInvite { #[derive(Deserialize)] pub struct NewUser { pub email: String, - pub password: String, + /// Required when `login_type` is `password` (the default), ignored otherwise. + pub password: Option, pub super_admin: bool, pub name: Option, pub company: Option, pub skip_email: Option, + /// `password`, `pending_oauth`, or a configured OAuth login client key. + pub login_type: Option, } #[derive(Deserialize)] @@ -1744,7 +1758,6 @@ struct ChangeUserEmail { /// `varchar(50)`, and `v2_job.permissioned_as` in a `varchar(55)`; every other email column is /// `varchar(255)`. The strictest of the two bounds is used for all of them. const SHORT_EMAIL_COLUMN_MAX_LEN: usize = 50; -const EMAIL_COLUMN_MAX_LEN: usize = 255; /// Move an account to a new email address, in place: the `password` row (and with it the /// instance-wide username, the role and the login type) is kept and every email-keyed row is @@ -2149,6 +2162,28 @@ async fn change_user_email( .execute(&mut *tx) .await?; + // An app draft carries a copy of the deployed policy, principal included. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2"#, + &new_principal, + &old_principal + ) + .execute(&mut *tx) + .await?; + + // A raw-app draft persists the address the client read back too. The deploy sends it beside + // the principal, where an address naming somebody else is rejected — and unlike a live read + // it never refreshes on its own. Same group guard as the deployed policy above, plus the + // `IS NULL` arm: without it the predicate is `NULL` for a draft with no principal, which is + // neither true nor false, so those rows would be skipped. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of_email' = $2 AND (value->'policy'->>'on_behalf_of' IS NULL OR value->'policy'->>'on_behalf_of' NOT LIKE 'g/%')"#, + &new_email, + &old_email + ) + .execute(&mut *tx) + .await?; + // A folder's default rules are an ordered array, first match wins, so the rewrite has to // preserve their order. A rule left on the old address makes `ensure_permissioned_as_exists` // reject the creation of every runnable the rule matches. @@ -2326,9 +2361,9 @@ async fn change_user_email( ) .await?; - // Read back inside the transaction: the address is derived at dispatch through a cache - // that nothing else evicts, so without this a job pushed in the next 60s would resolve - // the old address and with it the wrong superadmin flag and instance groups. + // Read back inside the transaction so this process can evict its own keys immediately. + // `notify_user_email_change` reaches every replica for the same change, but asynchronously, + // and this one is the replica that just served the request. let memberships = sqlx::query_scalar!("SELECT workspace_id FROM usr WHERE email = $1", &new_email) .fetch_all(&mut *tx) @@ -3159,6 +3194,506 @@ async fn impersonate( Ok((StatusCode::CREATED, token)) } +const LOGIN_LINK_DEFAULT_TTL_S: u32 = 600; +const LOGIN_LINK_MAX_TTL_S: u32 = 900; +const LOGIN_LINK_DEFAULT_RD: &str = "/user/workspaces"; +const LOGIN_LINK_EXPIRED_PAGE: &str = "/user/login_link_expired"; + +#[derive(Deserialize)] +pub struct NewLoginLink { + pub email: String, + pub expires_in_s: Option, + pub rd: Option, + /// Refuse to mint unless the account still has this login type: a caller re-entering an + /// account it created can require `pending_oauth`, so the link stops working once the + /// owner has set a password or signed in with a provider. + pub require_login_type: Option, +} + +#[derive(Serialize)] +pub struct LoginLink { + pub url: String, + pub expires_at: chrono::DateTime, +} + +/// A post-login destination is only ever a same-origin path: anything else would hand the +/// fresh session's first navigation to another host. Control characters are refused because +/// browsers strip tab/newline from a `Location` before parsing it, so `/\t/host` reads as +/// the protocol-relative `//host`. +fn same_origin_rd(rd: Option) -> Option { + rd.filter(|r| { + r.starts_with('/') + && !r.starts_with("//") + && !r.contains('\\') + && !r.chars().any(|c| c.is_ascii_control()) + }) +} + +#[cfg(test)] +mod same_origin_rd_tests { + use super::same_origin_rd; + + fn accepts(rd: &str) -> bool { + same_origin_rd(Some(rd.to_string())).is_some() + } + + #[test] + fn only_plain_same_origin_paths_pass() { + assert!(accepts("/")); + assert!(accepts("/user/workspaces?rd=%2Fx")); + assert!(!accepts("https://evil.example/")); + assert!(!accepts("//evil.example/")); + assert!(!accepts("/\\evil.example/")); + assert!(!accepts("/\t/evil.example/")); + assert!(!accepts("/x\r\nSet-Cookie: a=b")); + assert!(!accepts("user/workspaces")); + } +} + +/// Both provisioning writes reference `password(email)`; a typo'd address from the +/// provisioning script should read as "no such account", not as a foreign-key error. +async fn require_account( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + email: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", + email + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if !exists { + return Err(Error::NotFound(format!("no account for {email}"))); + } + Ok(()) +} + +fn login_link_redirect(location: String) -> Response { + ( + StatusCode::FOUND, + [ + ("location", location), + ("referrer-policy", "no-referrer".to_string()), + ], + ) + .into_response() +} + +/// Mint a single-use link that signs `email` in when opened. The row is not a `token`: +/// it can only ever become a session, and burning it needs no cache invalidation. +async fn create_login_link( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(nl): Json, +) -> Result<(StatusCode, Json)> { + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + + let email = nl.email.to_lowercase(); + let rd = match nl.rd { + Some(rd) => Some(same_origin_rd(Some(rd)).ok_or_else(|| { + Error::BadRequest("rd must be a same-origin path starting with /".to_string()) + })?), + None => None, + }; + let ttl = nl + .expires_in_s + .unwrap_or(LOGIN_LINK_DEFAULT_TTL_S) + .clamp(1, LOGIN_LINK_MAX_TTL_S); + + let mut tx = db.begin().await?; + let target = sqlx::query!( + "SELECT super_admin, devops, login_type FROM password WHERE email = $1 AND disabled = false", + &email + ) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + return Err(Error::NotFound(format!("no active account for {email}"))); + }; + // A link is a full session for its account; whoever holds the minting credential + // must not be able to turn it into an instance-wide role. + if target.super_admin || target.devops { + return Err(Error::BadRequest( + "login links cannot target superadmin or devops accounts".to_string(), + )); + } + if let Some(required) = nl.require_login_type.as_deref() { + if target.login_type != required { + return Err(Error::Generic( + StatusCode::CONFLICT, + format!( + "login_type_mismatch: {email} signs in with {}, not {required}", + target.login_type + ), + )); + } + } + + let token = rd_string(32); + let expires_at = chrono::Utc::now() + chrono::Duration::seconds(ttl as i64); + sqlx::query!( + "INSERT INTO login_link (token_hash, email, rd, expiration, created_by) + VALUES ($1, $2, $3, $4, $5)", + hash_token(&token), + &email, + rd, + expires_at, + &authed.email, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.login_link.create", + ActionKind::Create, + "global", + Some(&email), + Some([("expires_in_s", &ttl.to_string()[..])].into()), + ) + .await?; + tx.commit().await?; + + let url = format!( + "{}/api/auth/login_link/{}", + (**BASE_URL.load()).clone(), + token + ); + Ok((StatusCode::CREATED, Json(LoginLink { url, expires_at }))) +} + +#[derive(Deserialize)] +pub struct CloudTrialOfferUpdate { + pub email: String, + #[serde(default)] + pub consumed: bool, +} + +#[derive(Deserialize)] +pub struct OnboardingProfileUpdate { + pub email: String, + pub profile: serde_json::Value, +} + +#[derive(Serialize)] +pub struct OnboardingProfile { + pub profile: Option, +} + +/// Context the invite carried about this account's owner, written at provisioning. +/// Onboarding tailors itself from it (today: `touch_point` answers the source question +/// so it is never asked); everything degrades to the plain flow when absent. +async fn set_onboarding_profile( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(body): Json, +) -> Result { + if !*CLOUD_HOSTED { + return Err(Error::NotFound("cloud only".to_string())); + } + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + if !body.profile.is_object() { + return Err(Error::BadRequest( + "profile must be a JSON object".to_string(), + )); + } + let email = body.email.to_lowercase(); + let mut tx = db.begin().await?; + require_account(&mut tx, &email).await?; + sqlx::query!( + "INSERT INTO cloud_onboarding_profile (email, profile, created_by) VALUES ($1, $2, $3) + ON CONFLICT (email) DO UPDATE SET profile = EXCLUDED.profile", + &email, + body.profile, + &authed.email + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "users.onboarding_profile.set", + ActionKind::Update, + "global", + Some(&email), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("onboarding profile for {email} recorded")) +} + +async fn get_onboarding_profile( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + if !*CLOUD_HOSTED { + return Ok(Json(OnboardingProfile { profile: None })); + } + let profile = sqlx::query_scalar!( + "SELECT profile FROM cloud_onboarding_profile WHERE email = $1", + &authed.email + ) + .fetch_optional(&db) + .await?; + Ok(Json(OnboardingProfile { profile })) +} + +#[derive(Serialize)] +pub struct CloudTrialOffer { + pub offered: bool, +} + +/// What the customer portal answers when asked to sign a cloud account in and start its +/// pre-approved trial. +pub enum PortalTrialLogin { + /// Send the browser here: a short-lived portal login that starts the trial on landing. + LoginUrl(String), + /// The portal will not start one (a subscription exists, or it knows no offer); the + /// offer is spent and the browser goes to the portal home instead. + Unavailable { reason: String, portal_url: String }, +} + +async fn set_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(body): Json, +) -> Result { + if !*CLOUD_HOSTED { + return Err(Error::NotFound("cloud only".to_string())); + } + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + let email = body.email.to_lowercase(); + let mut tx = db.begin().await?; + require_account(&mut tx, &email).await?; + if body.consumed { + sqlx::query!( + "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + &email + ) + .execute(&mut *tx) + .await?; + } else { + // A consumed offer stays consumed: a trial or subscription already exists for it. + sqlx::query!( + "INSERT INTO cloud_trial_offer (email, created_by) VALUES ($1, $2) + ON CONFLICT (email) DO NOTHING", + &email, + &authed.email + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.cloud_trial_offer.set", + ActionKind::Update, + "global", + Some(&email), + Some([("consumed", if body.consumed { "true" } else { "false" })].into()), + ) + .await?; + tx.commit().await?; + Ok(format!( + "cloud trial offer for {email} {}", + if body.consumed { + "consumed" + } else { + "recorded" + } + )) +} + +async fn offered(db: &DB, email: &str) -> Result { + Ok(sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM cloud_trial_offer WHERE email = $1 AND consumed_at IS NULL)", + email + ) + .fetch_one(db) + .await? + .unwrap_or(false)) +} + +async fn get_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + if !*CLOUD_HOSTED { + return Ok(Json(CloudTrialOffer { offered: false })); + } + Ok(Json(CloudTrialOffer { + offered: offered(&db, &authed.email).await?, + })) +} + +/// Where the browser goes to start the pre-approved trial: a signed-in portal login, or +/// the portal's front page with the reason it could not start one. +#[derive(Serialize)] +pub struct CloudTrialOfferGo { + pub location: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// The one click that turns a cloud account's pre-approved offer into a trial: the portal +/// is asked for a login that starts it, and the browser is handed over. The portal is the +/// authority on whether the offer still stands; its refusal spends the offer here so the +/// sidebar stops advertising it. +async fn go_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + // The answer carries a signed-in portal login for this account: a credential for + // another system, and asking for it starts the trial. A script running as the offered + // user holds their identity through `$WM_TOKEN`, so a job token must not be able to + // fetch it and hand it to whoever wrote the script. It is a POST answered as JSON, not + // a redirecting GET, so a cross-site top-level navigation cannot start the trial with + // the SameSite=Lax session cookie either; the frontend navigates to `location` itself. + if authed.job_id.is_some() { + return Err(Error::NotAuthorized( + "This endpoint cannot be called with a job token ($WM_TOKEN).".to_string(), + )); + } + if !*CLOUD_HOSTED || !offered(&db, &authed.email).await? { + return Err(Error::NotFound( + "no pre-approved trial offer for this account".to_string(), + )); + } + let outcome = crate::users_oss::portal_cloud_trial_login(&authed.email).await?; + let (location, reason) = match outcome { + PortalTrialLogin::LoginUrl(url) => (url, None), + PortalTrialLogin::Unavailable { reason, portal_url } => (portal_url, Some(reason)), + }; + let mut tx = db.begin().await?; + if reason.is_some() { + sqlx::query!( + "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + &authed.email + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.cloud_trial_offer.go", + ActionKind::Execute, + "global", + Some(&authed.email), + Some([("outcome", reason.as_deref().unwrap_or("login"))].into()), + ) + .await?; + tx.commit().await?; + Ok(Json(CloudTrialOfferGo { location, reason })) +} + +#[derive(Deserialize)] +struct LoginLinkQuery { + rd: Option, +} + +async fn consume_login_link( + headers: axum::http::HeaderMap, + cookies: Cookies, + Extension(db): Extension, + Path(token): Path, + Query(query): Query, +) -> Result { + let bounce = |reason: &str| { + Ok(login_link_redirect(format!( + "{LOGIN_LINK_EXPIRED_PAGE}?reason={reason}" + ))) + }; + if token.len() != 32 { + return bounce("invalid"); + } + let t_hash = hash_token(&token); + // The account is unknown until the row is read, so only the global and per-IP tiers + // apply here; a 32-char random token leaves nothing for the per-account tier to guard. + windmill_common::login_rate_limit::check_and_increment_login_attempt( + &headers, + &t_hash[..TOKEN_PREFIX_LEN], + )?; + + let mut tx = db.begin().await?; + let link = sqlx::query!( + "UPDATE login_link SET consumed_at = now() + WHERE token_hash = $1 AND consumed_at IS NULL AND expiration > now() + RETURNING email, rd, created_by", + &t_hash + ) + .fetch_optional(&mut *tx) + .await?; + let Some(link) = link else { + let used = sqlx::query_scalar!( + "SELECT consumed_at IS NOT NULL AS \"used!\" FROM login_link WHERE token_hash = $1", + &t_hash + ) + .fetch_optional(&mut *tx) + .await?; + return bounce(match used { + Some(true) => "used", + Some(false) => "expired", + None => "invalid", + }); + }; + + // Re-checked at open time and locked through session creation: a promotion inside + // the link's window must not turn a link minted for an ordinary account into a + // privileged session. The bounce drops the transaction, so the link is not spent. + let target = sqlx::query!( + "SELECT super_admin, devops FROM password WHERE email = $1 AND disabled = false FOR UPDATE", + &link.email + ) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + return bounce("invalid"); + }; + if target.super_admin || target.devops { + return bounce("invalid"); + } + + let session = create_session_token(&link.email, false, None, false, &mut tx, cookies).await?; + audit_log( + &mut *tx, + &AuditAuthor { + email: link.email.clone(), + username: link.email.clone(), + username_override: None, + token_prefix: Some(safe_token_prefix(&session)), + }, + "users.login", + ActionKind::Create, + "global", + Some(&truncate_token(&session)), + Some( + [ + ("method", "login_link"), + ("minted_by", link.created_by.as_str()), + ] + .into(), + ), + ) + .await?; + tx.commit().await?; + + let rd = link + .rd + .or_else(|| same_origin_rd(query.rd)) + .unwrap_or_else(|| LOGIN_LINK_DEFAULT_RD.to_string()); + Ok(login_link_redirect(rd)) +} + #[derive(Deserialize)] pub struct ImpersonateServiceAccountRequest { pub username: String, @@ -3549,12 +4084,63 @@ async fn get_all_runnables( #[derive(Deserialize, Debug, Clone)] pub struct LoginUserInfo { pub email: Option, + /// OIDC `email_verified` claim where the provider sends one. + #[serde(default, deserialize_with = "deserialize_lenient_bool")] + pub email_verified: Option, pub name: Option, pub company: Option, pub preferred_username: Option, pub displayName: Option, } +/// Some providers (Cognito among them) send `email_verified` as the strings "true"/"false"; +/// a strict bool would reject their whole userinfo document and break login. +fn deserialize_lenient_bool<'de, D: serde::Deserializer<'de>>( + d: D, +) -> std::result::Result, D::Error> { + Ok(match Option::::deserialize(d)? { + Some(serde_json::Value::Bool(b)) => Some(b), + Some(serde_json::Value::String(s)) => match s.trim().to_ascii_lowercase().as_str() { + "true" => Some(true), + "false" => Some(false), + _ => None, + }, + _ => None, + }) +} + +#[cfg(test)] +mod login_user_info_tests { + use super::LoginUserInfo; + + fn email_verified(json: &str) -> Option { + serde_json::from_str::(json) + .unwrap() + .email_verified + } + + #[test] + fn email_verified_accepts_bool_and_stringified_bool() { + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":true}"#), + Some(true) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"true"}"#), + Some(true) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"false"}"#), + Some(false) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"maybe"}"#), + None + ); + assert_eq!(email_verified(r#"{"email":"a@b"}"#), None); + } +} + #[derive(Serialize)] struct InstanceUsernameInfo { username: String, diff --git a/backend/windmill-api-users/src/users_oss.rs b/backend/windmill-api-users/src/users_oss.rs index a42cce8405..a1eedd8563 100644 --- a/backend/windmill-api-users/src/users_oss.rs +++ b/backend/windmill-api-users/src/users_oss.rs @@ -26,3 +26,13 @@ pub async fn impersonate_service_account( "Service accounts require Windmill Enterprise Edition".to_string(), )) } + +#[cfg(not(feature = "private"))] +pub(crate) async fn portal_cloud_trial_login( + _email: &str, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::FeatureUnavailable( + "Starting a pre-approved trial from Windmill Cloud requires Windmill Enterprise Edition" + .to_string(), + )) +} diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 6ee55ed003..07bef364c8 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -19,6 +19,7 @@ use windmill_common::{ db::UserDB, error::JsonResult, jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE}, + queue_metrics::{read_queue_metrics_series, QueueMetricsSeries}, utils::{paginate, Pagination}, worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE}, workspaces::workspace_with_fork_ancestors, @@ -38,6 +39,8 @@ pub fn global_service() -> Router { ) .route("/get_default_tags", get(get_default_tags)) .route("/queue_metrics", get(get_queue_metrics)) + .route("/queue_metrics_series", get(get_queue_metrics_series)) + .route("/queue_status", get(get_queue_status)) .route("/queue_counts", get(get_queue_counts)) .route("/queue_running_counts", get(get_queue_running_counts)) .route( @@ -270,10 +273,16 @@ async fn get_queue_metrics( ) -> JsonResult> { require_devops_role(&db, &authed).await?; + // The API declares every `value` a number, so a climbing delay, stored as its head's wait + // start, is returned as the delay at the time of its sample. let queue_metrics = sqlx::query_as!( QueueMetric, "WITH queue_metrics as ( - SELECT id, value, created_at + SELECT id, created_at, + CASE WHEN jsonb_typeof(value) = 'object' + THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric) + ELSE value + END AS value FROM metrics WHERE id LIKE 'queue_%' AND created_at > now() - interval '14 day' @@ -289,6 +298,88 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } +#[derive(Deserialize)] +struct QueueMetricsSeriesQuery { + window_secs: Option, +} + +const QUEUE_METRICS_DEFAULT_WINDOW_SECS: i64 = 24 * 3600; +/// Retention of queue metrics, past which there is nothing left to read. +const QUEUE_METRICS_MAX_WINDOW_SECS: i64 = 14 * 24 * 3600; + +async fn get_queue_metrics_series( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult { + require_devops_role(&db, &authed).await?; + + let window = query + .window_secs + .unwrap_or(QUEUE_METRICS_DEFAULT_WINDOW_SECS) + .clamp(60, QUEUE_METRICS_MAX_WINDOW_SECS); + Ok(Json(read_queue_metrics_series(&db, window as f64).await?)) +} + +#[derive(Serialize)] +struct QueueTagStatus { + tag: String, + /// Jobs due for more than 3 seconds that no worker has picked up. + waiting: u32, + /// How long the job the next pull would take has been waiting, in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + delay: Option, + running: i64, + /// Workers that pinged in the last minute and pull this tag. + workers: i64, +} + +/// Every tag with jobs waiting or running, read live from the queue. A backlog on a tag no live +/// worker pulls waits for one to start: a worker group scaling up from zero, or none at all for a +/// tag nobody serves. +async fn get_queue_status( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_devops_role(&db, &authed).await?; + + let backlog = windmill_common::queue::get_queue_stats(&db).await?; + let backlog_tags = backlog.keys().cloned().collect::>(); + // A job's tag is resolved before it is queued (per-workspace and dedicated worker tags + // included), and the pull matches it exactly against the worker's tags, so containment is + // exact here too. + let rows = sqlx::query!( + "WITH running AS ( + SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag + ) + SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\", + (SELECT count(*) FROM worker_ping w + WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag] + ) AS \"workers!\" + FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag) + LEFT JOIN running r ON r.tag = t.tag + ORDER BY t.tag", + &backlog_tags[..], + ) + .fetch_all(&db) + .await?; + + Ok(Json( + rows.into_iter() + .map(|row| { + let stat = backlog.get(&row.tag); + QueueTagStatus { + waiting: stat.map_or(0, |s| s.count), + delay: stat.map(|s| s.delay), + running: row.running, + workers: row.workers, + tag: row.tag, + } + }) + .collect(), + )) +} + async fn get_queue_counts( authed: ApiAuthed, Extension(db): Extension, diff --git a/backend/windmill-api-workspaces/src/ai_session_backups.rs b/backend/windmill-api-workspaces/src/ai_session_backups.rs new file mode 100644 index 0000000000..8e7b75e1e2 --- /dev/null +++ b/backend/windmill-api-workspaces/src/ai_session_backups.rs @@ -0,0 +1,274 @@ +//! What the workspace key rotation, the workspace storage settings and the AI session backup +//! routes (`windmill-api/src/ai_sessions.rs`) share about the backups: the store they live +//! in, and what a rotation or a storage change deletes. +//! +//! The backups are ciphertext under the workspace key and live under a prefix named by a +//! generation the rotation bumps (`workspace_settings.ai_sessions_backup_generation`) in the +//! transaction that commits the new key. A rotation does not re-key them: once committed, +//! the routes read and write under the new generation's prefix and answer with its number +//! (`backup_generation`; the storage identity, `storage_id`, names the storage and does not +//! change), so every browser marks its sync state stale and pushes its sessions whole again +//! there, and every older generation, which nothing writes to any +//! more, is deleted off the request at leisure. Sessions no browser holds any more are lost, +//! which a rotation (a rare operation) accepts in exchange for having no key but the current +//! one to read with and nothing to rewrite in place. A generation is never reused, so no +//! deletion, however late, can touch live objects; a rotation that fails before its commit +//! bumps nothing and deletes nothing; two rotations racing serialize on the key row. +//! +//! A workspace without storage of its own keeps its backups in the instance object store +//! instead, under the same layout and key, while `ai_sessions_instance_storage_fallback` +//! allows it. Configuring a storage for such a workspace bumps the generation in the +//! transaction that sets it, so everything the workspace left in any instance store sits +//! under a generation the routes never read again: a later return to the instance store, +//! whichever it is by then, starts from a newer one. That is what lets a storage change +//! delete the older generations from the instance store without fencing against what +//! happens next, and a browser retire a removal owed to an instance store once the +//! workspace's own storage answered. + +use std::sync::Arc; + +use futures::{StreamExt, TryStreamExt}; +use windmill_common::error::{Error, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::DB; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, +}; +use windmill_object_store::{ + object_store_error_to_error, object_store_location, ObjectStoreResource, +}; +use windmill_types::s3::LargeFileStorage; + +/// The root of every AI session backup key in a workspace's storage. +pub const ROOT: &str = "windmill_ai_sessions"; +/// The push body cap: no object written through the routes is larger. One that is was +/// planted by whoever holds the bucket's credentials, and is left unread. +pub const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; +/// The storage name the workspace's backups in the instance store count under in its +/// storage usage, next to `_default_` and the secondary storages. +pub const FALLBACK_STORAGE: &str = "_ai_sessions_fallback_"; + +const IO_CONCURRENCY: usize = 8; + +/// The prefix of one generation's objects: `windmill_ai_sessions/{w_id}/g{generation}/`. +pub fn generation_prefix(w_id: &str, generation: i64) -> String { + format!("{ROOT}/{w_id}/g{generation}") +} + +/// The prefix of everything the workspace ever backed up, whatever the generation. +fn workspace_prefix(w_id: &str) -> ObjectPath { + ObjectPath::from(format!("{ROOT}/{w_id}")) +} + +/// Names the storage the backups are in, by what locates its objects (endpoint, region, +/// bucket; never the credentials, which rotate), so a browser tells that its sync state was +/// recorded against another storage; the generation, answered alongside, tells it a +/// rotation happened in this one. +pub fn storage_id(resource: &ObjectStoreResource) -> String { + calculate_hash(&object_store_location(resource))[..16].to_string() +} + +/// Where a workspace's backups live: its primary storage, or the instance object store +/// standing in for it. +pub struct BackupStore { + pub store: Arc, + pub storage_id: String, + pub fallback: bool, +} + +/// The instance object store, for a workspace without storage of its own: loaded from +/// settings that say where its objects are, and not turned off by +/// `ai_sessions_instance_storage_fallback`, which is on unless set to false. Named like a +/// workspace storage, by that location, in a namespace of its own. Never on the Pro plan, +/// checked on every call: a store loaded before a switch to Pro stays loaded. Never in a +/// build without `private`, which has neither workspace storage nor the quota the fallback +/// counts toward. +/// +/// Authorizes nothing, and the store reaches every workspace's objects: the caller must have +/// authorized the user for the workspace and keep what it reads and writes under that +/// user's prefix in it, as the backup routes do. +pub async fn fallback_store(db: &DB) -> Result> { + #[cfg(not(feature = "private"))] + { + let _ = db; + Ok(None) + } + #[cfg(feature = "private")] + { + if matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro + ) { + return Ok(None); + } + let Some((store, Some(location))) = + windmill_object_store::get_object_store_with_location().await + else { + return Ok(None); + }; + let setting = windmill_common::global_settings::load_value_from_global_settings( + db, + windmill_common::global_settings::AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING, + ) + .await?; + if matches!(setting, Some(serde_json::Value::Bool(false))) { + return Ok(None); + } + Ok(Some(BackupStore { + storage_id: calculate_hash(&format!("instance:{location}"))[..16].to_string(), + store, + fallback: true, + })) + } +} + +/// The workspace's primary storage, resolved without a caller: a rotation runs its deletion +/// off its own request, and the retention sweep off any. The caller must be the server +/// itself; nothing here checks who asks. +pub async fn primary_store(db: &DB, w_id: &str) -> Result> { + let Some(lfs_json) = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten() else { + return Ok(None); + }; + let lfs: LargeFileStorage = serde_json::from_value(lfs_json) + .map_err(|e| Error::internal_err(format!("parsing large_file_storage: {e}")))?; + let resource_value = if matches!(lfs, LargeFileStorage::FilesystemStorage(_)) { + serde_json::Value::Null + } else { + let path = lfs.get_s3_resource_path(); + let path = path.strip_prefix("$res:").unwrap_or(path); + windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{path}")), + w_id, + db, + ) + .await? + }; + let resource = windmill_object_store::lfs_to_object_store_resource(&lfs, resource_value)?; + Ok(Some(BackupStore { + store: windmill_object_store::build_object_store_client(&resource).await?, + storage_id: storage_id(&resource), + fallback: false, + })) +} + +/// The store the workspace's backups live in, resolved without a caller. +async fn workspace_store(db: &DB, w_id: &str) -> Result> { + if let Some(primary) = primary_store(db, w_id).await? { + return Ok(Some(primary)); + } + fallback_store(db).await +} + +/// The generation an object key sits under, `None` for a key of no generation (an older +/// layout), which counts as older than any. +fn generation_of(w_id: &str, key: &ObjectPath) -> Option { + key.as_ref() + .strip_prefix(&format!("{ROOT}/{w_id}/g"))? + .split('/') + .next()? + .parse() + .ok() +} + +/// Deletes, as the listing streams, every object of the workspace's backups in the store +/// from a generation older than `current`. +async fn delete_older(store: &Arc, w_id: &str, current: i64) -> Result<()> { + store + .list(Some(&workspace_prefix(w_id))) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { + return Ok(()); + } + match store.delete(&meta.location).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + }) + .await +} + +/// Deletes, off the request, every object of the workspace's backups from a generation +/// older than `current`, once the rotation that made `current` the generation has +/// committed: nothing writes there any more but a push that resolved its prefix before the +/// commit, junk the browser's next push of that session rewrites under the current prefix, +/// as is anything a deletion cut short left behind. For the rotation route, which +/// authorized its caller as a superadmin. +pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { + tokio::spawn(async move { + let store = match workspace_store(&db, &w_id).await { + Ok(Some(store)) => store.store, + Ok(None) => return, + Err(e) => { + tracing::warn!("older AI session backups of {w_id} left in place: {e:#}"); + return; + } + }; + match delete_older(&store, &w_id, current).await { + Ok(()) => { + tracing::info!("deleted the AI session backups of {w_id} older than g{current}") + } + Err(e) => tracing::warn!("deleting the older AI session backups of {w_id}: {e:#}"), + } + }); +} + +/// Deletes, off the request, what the workspace's backups left in the instance store under +/// a generation older than `current`, the one a storage settings change committed. Nothing +/// reads there: the routes use the workspace's own storage, or, back in the instance store, +/// `current` or a newer generation, since configuring a storage over the fallback bumped +/// it. So it runs whatever the storage is now and whatever the setting says (copies from +/// when it was on may be there), and a deletion that is slow, cut short or overtaken by a +/// later change deletes nothing live. For the storage settings route, which authorized its +/// caller as a workspace admin. +pub(crate) fn spawn_delete_fallback(w_id: String, current: i64) { + tokio::spawn(async move { + let Some(instance) = windmill_object_store::get_object_store().await else { + return; + }; + match delete_older(&instance, &w_id, current).await { + Ok(()) => tracing::info!( + "deleted the AI session backups of {w_id} older than g{current} from the instance store" + ), + Err(e) => tracing::warn!( + "deleting the AI session backups of {w_id} from the instance store: {e:#}" + ), + } + }); +} + +/// The bytes of the workspace's backups in the instance store, for its storage usage while +/// it has no storage of its own (once it has one nothing writes there, and the change +/// deleted what was): `None` when it has one, when there is no instance store, or when +/// there is nothing, so no empty usage entry shows up. Whether the setting is on or off, +/// since copies from when it was on may be there. +/// +/// Authorizes nothing: for the storage usage recount, which reports a total for the +/// workspace it was run for and hands out nothing it read. +pub async fn fallback_bytes(db: &DB, w_id: &str) -> Result> { + let has_storage = sqlx::query_scalar!( + r#"SELECT large_file_storage IS NOT NULL AS "has_storage!" FROM workspace_settings WHERE workspace_id = $1"#, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if has_storage { + return Ok(None); + } + let Some(instance) = windmill_object_store::get_object_store().await else { + return Ok(None); + }; + let mut total: i64 = 0; + let mut stream = instance.list(Some(&workspace_prefix(w_id))); + while let Some(meta) = stream.next().await { + total += meta.map_err(object_store_error_to_error)?.size as i64; + } + Ok((total > 0).then_some(total)) +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index 017c9702a5..22f2a2c5bb 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,6 +1,8 @@ +#[cfg(feature = "parquet")] +pub mod ai_session_backups; +pub mod data_metrics; pub mod datatable_migrations; pub mod deployment_requests; -pub mod data_metrics; pub mod workspaces; pub mod workspaces_extra; pub mod workspaces_oss; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c9343840d2..3e48e7baca 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -227,6 +227,10 @@ pub fn global_service() -> Router { .route("/list", get(list_workspaces)) .route("/users", get(user_workspaces)) .route("/session_workspace_status", post(session_workspace_status)) + .route( + "/session_workspace_retention", + post(session_workspace_retention), + ) .route("/create", post(create_workspace)) .route("/create_fork", post(deprecated_create_workspace_fork)) .route("/exists", post(exists_workspace)) @@ -910,12 +914,15 @@ fn redact_git_sync_webhook_secrets(git_sync: &mut serde_json::Value) { } /// Zero the server-owned auto-pull fields (webhook id/secret/url/error, synced -/// sha, last pull status) on a client-supplied `AutoPullSettings`. The client only -/// controls `enabled` / `mode` / `poll_interval_s`; the rest is written by the -/// server (webhook creation, poller) and must never be trusted from the request — -/// otherwise a caller could inject a webhook id/secret or fake sync state. -fn clear_client_supplied_auto_pull_state( +/// sha, last pull status) on a client-supplied `AutoPullSettings`, and stamp who +/// pulls run as: `saver_email` while auto pull is on. The client only controls +/// `enabled` / `mode` / `poll_interval_s`; the rest is written by the server (webhook +/// creation, poller, this save) and must never be trusted from the request — +/// otherwise a caller could inject a webhook id/secret, fake sync state, or pick who +/// pulls run as. +fn sanitize_client_auto_pull( auto_pull: &mut windmill_common::workspaces::AutoPullSettings, + saver_email: &str, ) { auto_pull.webhook_id = None; auto_pull.webhook_secret = None; @@ -923,6 +930,28 @@ fn clear_client_supplied_auto_pull_state( auto_pull.webhook_error = None; auto_pull.last_synced_sha = std::collections::HashMap::new(); auto_pull.last_pull_status = None; + auto_pull.enabled_by = auto_pull.enabled.then(|| saver_email.to_string()); +} + +#[cfg(test)] +mod sanitize_client_auto_pull_tests { + use windmill_common::workspaces::AutoPullSettings; + + #[test] + fn a_save_stamps_the_saver_over_any_client_supplied_stamp() { + let mut ap = AutoPullSettings { + enabled: true, + enabled_by: Some("forged@example.com".to_string()), + ..Default::default() + }; + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by.as_deref(), Some("saver@example.com")); + + ap.enabled = false; + ap.enabled_by = Some("forged@example.com".to_string()); + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by, None, "auto pull off carries no stamp"); + } } /// Whether a git-sync repository tracking `tracked` rules out `label_branch` as a dev workspace's @@ -2068,6 +2097,17 @@ async fn edit_large_file_storage_config( serde_json::to_value::(lfs_config) .map_err(|err| Error::internal_err(err.to_string()))?; + // A workspace whose AI session backups fell back to the instance store leaves it + // here: the generation moves on, so nothing it left in any instance store is read + // again, whichever one a later return to the fallback finds (`ai_session_backups`). + sqlx::query!( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 \ + WHERE workspace_id = $1 AND large_file_storage IS NULL", + &w_id + ) + .execute(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", serialized_lfs_config, @@ -2083,8 +2123,23 @@ async fn edit_large_file_storage_config( .execute(&mut *tx) .await?; } + let backups_generation = sqlx::query_scalar!( + "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await?; tx.commit().await?; + // Read by nothing any more, whatever the storage is now: what the AI session backups + // left in the instance store under a generation older than the one just committed. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_fallback(w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Trigger git sync for large file storage changes handle_deployment_metadata( &authed.email, @@ -3983,9 +4038,10 @@ async fn edit_git_sync_config( // stay clean. for repo in git_sync_settings.repositories.iter_mut() { if let Some(ap) = repo.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } repo.open_pr_error = None; + repo.credential = None; } reject_parent_only_git_sync_settings_on_fork( &db, @@ -4086,6 +4142,7 @@ async fn edit_git_sync_config( continue; }; repo.open_pr_error = old.open_pr_error.clone(); + repo.credential = old.credential.clone(); if let (Some(new_ap), Some(old_ap)) = (repo.auto_pull.as_mut(), old.auto_pull.as_ref()) { @@ -4131,6 +4188,7 @@ async fn edit_git_sync_config( .flatten() .and_then(|v| serde_json::from_value(v).ok()); let removed_webhooks: Vec<(String, i64)> = existing + .as_ref() .map(|e| { e.repositories .iter() @@ -4164,6 +4222,19 @@ async fn edit_git_sync_config( // `sync_repo_webhook` writes back the webhook fields it changes itself: // the remote hook and the record of it have to move together, so // persisting them out here would let one land without the other. + // Before the webhook reconcile, which decides whether this repo can have + // one from the credential this records. Also puts a short-lived or + // under-scoped token in front of the operator while they are still on the + // settings page, rather than when it expires. + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); @@ -4214,9 +4285,10 @@ async fn edit_git_sync_repository( // existing repo re-derives it from the DB (carried over below) and a new one // starts clean. if let Some(ap) = new_config.repository.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } new_config.repository.open_pr_error = None; + new_config.repository.credential = None; reject_parent_only_git_sync_settings_on_fork( &db, &w_id, @@ -4341,6 +4413,7 @@ async fn edit_git_sync_repository( // from the UI cannot revert what the poller/webhook layer wrote. let mut updated = new_config.repository; updated.open_pr_error = existing_repo.open_pr_error.clone(); + updated.credential = existing_repo.credential.clone(); match (updated.auto_pull.as_mut(), existing_repo.auto_pull.as_ref()) { (Some(new_ap), Some(old_ap)) => { new_ap.last_synced_sha = old_ap.last_synced_sha.clone(); @@ -4398,6 +4471,15 @@ async fn edit_git_sync_repository( .iter_mut() .find(|r| r.git_repo_resource_path == new_config.git_repo_resource_path) { + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); } @@ -4537,6 +4619,10 @@ async fn delete_git_sync_repository( } } + // The stored credential is deliberately left alone: it belongs to the git + // repository resource, which this endpoint does not delete, and the resource + // still authenticates with it for connection tests and commit lookups. + // Trigger git sync for repository deletion handle_deployment_metadata( &authed.email, @@ -4660,6 +4746,9 @@ async fn edit_guest_access( Json(EditGuestAccess { guest_access_enabled }): Json, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + if guest_access_enabled { + windmill_common::workspaces::require_guest_support()?; + } let mut tx = db.begin().await?; sqlx::query!( @@ -4716,6 +4805,11 @@ async fn edit_guest_jwt_key( "Set a PEM public key or a JWKS URL, not both".to_string(), )); } + // Clearing stays allowed wherever guests are: a key nobody can use is still worth + // removing. + if public_key.is_some() || jwks_url.is_some() { + windmill_common::workspaces::require_guest_support()?; + } if let Some(pem) = public_key.as_deref() { windmill_common::guest_jwt::decoding_key_from_pem(pem)?; } @@ -5275,6 +5369,17 @@ async fn set_encryption_key( let mut tx = db.begin().await?; + // Under the row's lock, so two rotations racing serialize and each sees the key the + // other committed. The AI session backups in the workspace storage live under a prefix + // named by a generation this bumps (with the key, in this transaction) rather than + // being re-keyed; the older generations are deleted once this one has committed (see + // `ai_session_backups`). The same key set again is no rotation to them. + let previous_key: String = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud' FOR UPDATE", + ) + .bind(&w_id) + .fetch_one(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), @@ -5282,6 +5387,18 @@ async fn set_encryption_key( ) .execute(&mut *tx) .await?; + let backups_generation: Option = if previous_key != request.new_key { + sqlx::query_scalar( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 WHERE workspace_id = $1 \ + RETURNING ai_sessions_backup_generation", + ) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await? + } else { + None + }; let mut reencrypted_secret_paths: Vec = Vec::new(); if !request.skip_reencrypt.unwrap_or(false) { @@ -5338,6 +5455,15 @@ async fn set_encryption_key( // Invalidate the cache only after the transaction has committed WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Nothing writes under the older generations any more; the browsers push their + // sessions again under the new one. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_older(db.clone(), w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Build the batch: one event for the encryption key itself plus one per // re-encrypted secret variable. The batch entrypoint dispatches a single // git-sync job per repo carrying all items, so repos with Secrets sync @@ -5507,6 +5633,14 @@ struct SessionWorkspaceStatusRequest { workspace_ids: Vec, } +/// `ai_config.sessions_retention_days` as stored, `None` when unset or not a count of days. +pub fn sessions_retention_days(value: Option<&serde_json::Value>) -> Option { + value + .and_then(|v| v.as_u64()) + .filter(|days| *days >= 1) + .and_then(|days| u32::try_from(days).ok()) +} + /// Reconciliation support for client-side AI sessions, which the backend cannot touch /// directly. The client posts the workspace ids its sessions reference and uses the /// per-id status to keep sessions in sync with workspace lifecycle: `deleted` (no row, or @@ -5556,6 +5690,42 @@ async fn session_workspace_status( Ok(Json(statuses)) } +/// The AI session retention a browser deletes its local copies by (docs/ai-session-backups.md). +/// Its own route, not a field on the status above, whose shape an older tab still reads. Unlike +/// a status, it answers only for a workspace this caller can be authed into: a setting is the +/// workspace's to tell, so a disabled membership gets none though its sessions still reconcile. +async fn session_workspace_retention( + Extension(db): Extension, + authed: ApiAuthed, + Json(req): Json, +) -> JsonResult> { + if req.workspace_ids.len() > 1000 { + return Err(Error::BadRequest( + "Too many workspace ids (max 1000)".to_string(), + )); + } + let email = &authed.email; + let is_superadmin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; + let rows = sqlx::query!( + "SELECT workspace_settings.workspace_id AS \"id!\", + workspace_settings.ai_config->'sessions_retention_days' AS retention + FROM workspace_settings + LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2 + WHERE workspace_settings.workspace_id = ANY($1) + AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + &req.workspace_ids[..], + email, + is_superadmin, + ) + .fetch_all(&db) + .await?; + let days = rows + .into_iter() + .filter_map(|r| sessions_retention_days(r.retention.as_ref()).map(|days| (r.id, days))) + .collect(); + Ok(Json(days)) +} + /// The instance critical alert channels belong to the instance operator, who on cloud is /// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run /// throwaway copies of their parent's runnables, so instance-wide operational alerting must @@ -6017,7 +6187,7 @@ async fn clone_triggers_and_schedules( path, route_path, route_path_key, script_path, is_flow, workspace_id, edited_by, edited_at, extra_perms, authentication_method, http_method, static_asset_config, is_static_website, workspaced_route, wrap_body, - raw_string, authentication_resource_path, summary, description, + raw_string, allowed_origins, authentication_resource_path, summary, description, error_handler_path, error_handler_args, retry, request_type, mode, permissioned_as, labels ) @@ -6025,7 +6195,7 @@ async fn clone_triggers_and_schedules( path, route_path, route_path_key, script_path, is_flow, $1, edited_by, edited_at, extra_perms, authentication_method, http_method, static_asset_config, is_static_website, workspaced_route, wrap_body, - raw_string, authentication_resource_path, summary, description, + raw_string, allowed_origins, authentication_resource_path, summary, description, error_handler_path, error_handler_args, retry, request_type, 'disabled'::TRIGGER_MODE, permissioned_as, labels FROM http_trigger @@ -6306,9 +6476,10 @@ async fn update_workspace_settings( // Auto-pull and fork PRs are parent-owned and must not be inherited: // the fork would otherwise carry the parent's webhook id (turning off // auto-pull on the fork would delete the parent's webhook). A fork - // still inherits the push-direction config and the installation. - // Repo → fork sync is driven by the parent's webhook/poller - // (`sync_forks`), which routes the fork's `wm-fork/**` branch into it. + // still inherits the push-direction config, the installation, and the + // recorded credential status, which describes the repository rather + // than belonging to either workspace and would otherwise leave the + // fork unqualified for managed features until its first check. r.auto_pull = None; r.fork_open_prs = false; r.open_pr_error = None; @@ -6426,8 +6597,8 @@ async fn clone_resource_types( target_workspace_id: &str, ) -> Result<()> { sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset) - SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset + "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name) + SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", source_workspace_id, @@ -6627,7 +6798,7 @@ async fn clone_scripts( } /// The parsed dbt graph a deployed script carries: its models, their SQL and -/// tests, and the `ref()` lineage between them. +/// tests, and the `ref()` and column-level lineage between them. /// /// Keyed on (workspace_id, script_path, script_hash), and the fork keeps every /// script's hash, so each row moves across as itself. @@ -6645,11 +6816,11 @@ async fn clone_dbt_graph( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at) + columns, column_schema, freshness, raw_code, original_file_path, ingested_at) SELECT $2, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at + columns, column_schema, freshness, raw_code, original_file_path, ingested_at FROM dbt_node WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", source_workspace_id, @@ -6669,6 +6840,24 @@ async fn clone_dbt_graph( ) .execute(&mut **tx) .await?; + // Column lineage travels with the rest of the graph, and it has to: the + // snapshot's digest covers it, so a fork missing these rows recomputes the + // digest the source stored, matches, and stores nothing — leaving the + // lineage gone until someone redeploys, which is the failure this whole + // function exists to prevent. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind, + ingested_at) + SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column, + child_unique_id, child_column, lineage_kind, ingested_at + FROM dbt_column_edge + WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + source_workspace_id, + target_workspace_id + ) + .execute(&mut **tx) + .await?; sqlx::query!( "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest, relation_root_at_last_ingest, ingested_at) @@ -7282,8 +7471,8 @@ async fn clone_workspace_runnable_dependencies( ) -> Result<()> { // Clone workspace_runnable_dependencies sqlx::query!( - "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path) - SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path) + SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path FROM workspace_runnable_dependencies WHERE workspace_id = $2", target_workspace_id, diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index b2883e5f9a..e75f0d1ef2 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -113,7 +113,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 3c816d03d8..8f188bf763 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.804.0 + version: 1.812.0 title: Windmill API contact: @@ -293,6 +293,108 @@ paths: items: $ref: "#/components/schemas/AuditLog" + /w/{workspace}/trash/list: + get: + summary: list the workspace trashbin (requires admin privilege) + operationId: listTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: item_kind + in: query + description: > + only return items of this kind: script, flow, app, schedule, variable, + resource, or a trigger kind such as http_trigger + schema: + type: string + - name: page + in: query + description: which page to return (starts at 0, default 0) + schema: + type: integer + - name: per_page + in: query + description: number of items to return for a given page (default 100, max 1000) + schema: + type: integer + responses: + "200": + description: the trashed items, most recently deleted first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TrashItem" + + /w/{workspace}/trash/get/{id}: + get: + summary: get a trashed item with the data it was deleted with (requires admin privilege) + operationId: getTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: the trashed item + content: + application/json: + schema: + $ref: "#/components/schemas/TrashItemWithData" + + /w/{workspace}/trash/restore/{id}: + post: + summary: restore a trashed item to its path (requires admin privilege) + operationId: restoreTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item restored + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/delete/{id}: + delete: + summary: permanently delete a trashed item (requires admin privilege) + operationId: permanentlyDeleteTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item permanently deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/empty: + post: + summary: permanently delete every item in the workspace trashbin (requires admin privilege) + operationId: emptyTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: trashbin emptied + content: + text/plain: + schema: + type: string + /auth/login: post: security: [] @@ -406,6 +508,28 @@ paths: "400": description: SMTP not configured + /auth/login_link/{token}: + get: + security: [] + summary: consume a single-use login link, set the session cookie and redirect + operationId: consumeLoginLink + tags: + - user + parameters: + - name: token + in: path + required: true + schema: + type: string + - name: rd + in: query + required: false + schema: + type: string + responses: + "302": + description: redirected to the post-login destination, or to /user/login_link_expired when the link is used, expired or unknown + /auth/reset_password: post: security: [] @@ -623,9 +747,14 @@ paths: skip_email: type: boolean description: Skip sending email notifications to the user + login_type: + type: string + description: >- + password (default, requires `password`), pending_oauth (no credential + until the first OAuth login proving the address adopts the account), or + a configured OAuth login client key required: - email - - password - super_admin responses: "201": @@ -1148,6 +1277,37 @@ paths: - archived - deleted + /workspaces/session_workspace_retention: + post: + summary: get the AI session retention of workspaces referenced by client-side sessions + operationId: getSessionWorkspaceRetention + tags: + - workspace + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + workspace_ids: + type: array + items: + type: string + required: + - workspace_ids + responses: + "200": + description: >- + map of workspace id to its `ai_config.sessions_retention_days`; a workspace + without a retention, or one the caller cannot be authenticated into, is absent + content: + application/json: + schema: + type: object + additionalProperties: + type: integer + /w/{workspace}/workspaces/get_as_superadmin: get: summary: get workspace as super admin (require to be super admin) @@ -2741,7 +2901,7 @@ paths: /w/{workspace}/github_app/token: post: - summary: get github app token + summary: get the git credential for a git-sync job (GitHub App token, or the credential stored for the repository) operationId: getGithubAppToken tags: - workspace @@ -2761,7 +2921,7 @@ paths: - job_token responses: "200": - description: github app token + description: git credential content: application/json: schema: @@ -2905,6 +3065,131 @@ paths: "200": description: Successfully imported the installation + /w/{workspace}/git_sync/gitlab/projects: + post: + tags: + - Git Sync + summary: List the GitLab projects a token can sync + description: >- + Lists the projects the supplied GitLab token can push to, so a git + repository resource can be filled in without hand-writing a project + path. The token is used for this call only and is never stored. + Requires workspace admin. + operationId: listGitlabProjects + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + base_url: + type: string + description: The GitLab instance, e.g. https://gitlab.com + token: + type: string + description: A project access token with the api scope, or a group token that reaches the project + search: + type: string + description: Narrow the list to projects matching this text + required: + - base_url + - token + responses: + "200": + description: the projects the token can sync + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/GitlabProject" + + /w/{workspace}/git_sync/credential/origin: + get: + tags: + - Git Sync + summary: Where a repository's credential comes from + description: >- + Whether Windmill holds this repository's access token, and which host it + talks to. `held` means this workspace stores it, `borrowed` means an + ancestor does and it is not this workspace's to replace. Both absent + means the repository authenticates with whatever its URL carries. + Returns no secret. Requires workspace admin. + operationId: getCredentialOrigin + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: >- + Path of the git repository resource, with or without the `$res:` + prefix. A path rather than a URL, because a resource URL may carry a + token and a URL in a query string lands in logs. + schema: + type: string + responses: + "200": + description: where the credential comes from + content: + application/json: + schema: + type: object + properties: + origin: + type: string + enum: + - held + - borrowed + provider: + type: string + enum: + - gitlab + + /w/{workspace}/git_sync/credential: + post: + tags: + - Git Sync + summary: Store the credential for a git repository + description: >- + Stores the access token a git repository authenticates with, so it does + not have to be written into the repository URL or a workspace variable. + The token is write-only: it is served only to a git-sync job that + presents its own job token, and a fork of this workspace reads this + copy instead of holding one of its own. Requires workspace admin. + operationId: setGitCredential + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + repo_url: + type: string + description: >- + The repository the credential is for, and the key it is + stored under. It is served for this repository and no other, + so repointing a resource elsewhere cannot carry the token + along. + token: + type: string + description: The access token, as pasted + required: + - repo_url + - token + responses: + "200": + description: the credential was stored + content: + text/plain: + schema: + type: string + /w/{workspace}/github_app/ghes_installation_callback: post: summary: GHES installation callback @@ -5788,7 +6073,8 @@ paths: account; the `guest` app execution mode admits them. Off by default. Re-read where a guest session is minted and at the auth door on every guest request, so turning it off takes effect immediately, for sessions already issued and for - apps whose policy already says `guest`. + apps whose policy already says `guest`. Turning it *on* is refused with a 400 + where guests are unavailable (the shared cloud); turning it off always works. operationId: editGuestAccess tags: - workspace @@ -5823,7 +6109,8 @@ paths: URL, at most one. Both empty clears the workspace key; off cloud, verification then falls back to the instance issuer (`JWT_EXT_JWKS_URL`) if one is set, else no guest JWT is accepted (`guest_access_enabled` is the on/off switch). Workspace-admin gated. - The key is validated before it is stored. + The key is validated before it is stored. Setting a key is refused with a 400 where + guests are unavailable (the shared cloud); clearing one always works. operationId: editGuestJwtKey tags: - workspace @@ -6257,6 +6544,172 @@ paths: schema: type: string + /users/login_links: + post: + summary: mint a single-use login link for an account (require superadmin) + operationId: createLoginLink + tags: + - user + requestBody: + description: target account and link options + required: true + content: + application/json: + schema: + type: object + required: + - email + properties: + email: + type: string + expires_in_s: + type: integer + description: link lifetime in seconds, at most 900 (default 600) + rd: + type: string + description: same-origin path the browser lands on after login (default /user/workspaces) + require_login_type: + type: string + description: >- + mint only while the account still has this login type (for example + pending_oauth), so a link stops working once the owner has set a password + or signed in with a provider + responses: + "201": + description: login link minted + content: + application/json: + schema: + type: object + required: + - url + - expires_at + properties: + url: + type: string + expires_at: + type: string + format: date-time + "409": + description: the account does not have the required login type + + /users/cloud_trial_offer: + post: + summary: record or consume a pre-approved self-hosted trial offer for a cloud account (require superadmin, cloud only) + operationId: setCloudTrialOffer + tags: + - user + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - email + properties: + email: + type: string + consumed: + type: boolean + description: mark the offer used (a trial or subscription now exists) instead of recording it + responses: + "200": + description: offer recorded or consumed + content: + text/plain: + schema: + type: string + get: + summary: whether the signed-in account holds an unconsumed pre-approved self-hosted trial offer (cloud only) + operationId: getCloudTrialOffer + tags: + - user + responses: + "200": + description: offer state + content: + application/json: + schema: + type: object + required: + - offered + properties: + offered: + type: boolean + + /users/cloud_trial_offer/go: + post: + summary: start the signed-in account's pre-approved self-hosted trial on the customer portal (cloud only). A POST answered as JSON rather than a redirecting GET, so a cross-site navigation cannot start it; the browser navigates to `location` itself + operationId: goCloudTrialOffer + tags: + - user + responses: + "200": + description: where to go — the customer portal signed in with the trial being started, or the portal home with the reason the offer could not be used + content: + application/json: + schema: + type: object + required: + - location + properties: + location: + type: string + reason: + type: string + description: present when the portal refused (e.g. the account already has a subscription); the offer is then spent + "403": + description: called with a job token + "404": + description: no offer for this account + + /users/onboarding_profile: + post: + summary: record the invite context an account's onboarding tailors itself from (require superadmin, cloud only) + operationId: setOnboardingProfile + tags: + - user + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - email + - profile + properties: + email: + type: string + profile: + type: object + description: free-form context from the invite, every key optional. The frontend reads `touch_point` (answers onboarding's source question), `company` and `workspace_name` (prefill the first workspace's name), `hub_projects` (slugs surfaced first on an empty workspace), `tools` (integrations, used to pick hub projects when none are named) and `starter_prompts` (`[{label, prompt}]`, replacing the home page's example prompts); unknown keys are kept and ignored + responses: + "200": + description: profile recorded + content: + text/plain: + schema: + type: string + get: + summary: the invite context recorded for the signed-in account, if any (cloud only) + operationId: getOnboardingProfile + tags: + - user + responses: + "200": + description: the profile, or null when none was recorded + content: + application/json: + schema: + type: object + properties: + profile: + type: object + nullable: true + additionalProperties: true + /users/tokens/delete/{token_prefix}: delete: summary: delete token @@ -8528,6 +8981,84 @@ paths: items: type: string + /w/{workspace}/resources/type/resource_counts: + get: + summary: count the workspace's resources per resource type + operationId: listResourceCountsByType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: resource count per resource_type + content: + application/json: + schema: + type: array + items: + type: object + properties: + resource_type: + type: string + count: + type: integer + required: + - resource_type + - count + + /w/{workspace}/resources/type/hub/info: + get: + summary: list what the hub knows about its resource types + operationId: listHubResourceTypeInfo + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: each hub resource type with its integration and pick count, empty if the hub answers neither read + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + app: + description: the integration the resource type belongs to, which is not always its own name + type: string + picks: + type: integer + required: + - name + - app + - picks + + /w/{workspace}/resources/type/hub/pick/{name}: + post: + summary: record a hub resource type pick + operationId: pickHubResourceType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Name" + responses: + "200": + description: whether the hub recorded the pick + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: + - success + /w/{workspace}/npm_proxy/config: get: summary: get npm proxy configuration @@ -8777,6 +9308,13 @@ paths: properties: name: type: string + picks: + description: how often the integration has been picked, absent on a hub that does not count picks + type: integer + display_name: + description: the label the hub curates for the integration, null or absent where it names none + type: string + nullable: true required: - name @@ -8944,7 +9482,8 @@ paths: description: >- The custom-path counterpart of `getGuestEntry`. Unauthenticated; 404 unless the app's execution mode is `guest` AND its workspace has - `guest_access_enabled` AND the instance has not set `guest_access_disabled`. + `guest_access_enabled` AND the instance has not set `guest_access_disabled`, + and never on a deployment where guests are unavailable (the shared cloud). Returns the workspace too, since a custom URL may not carry it. operationId: getGuestEntryByCustomPath tags: @@ -11564,6 +12103,25 @@ paths: items: type: string + /w/{workspace}/flows/list_paths_linking_agent/{path}: + get: + summary: list flow paths with a step linked to a saved agent + operationId: listFlowPathsLinkingAgent + tags: + - flow + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: paths of the flows linking the `ai_agent` resource, as of their last deploy + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/flows/get/v/{version}: get: summary: get flow version @@ -12687,6 +13245,288 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai/sessions/list: + get: + summary: list the calling user's AI session backups in the workspace object storage + operationId: listAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: backups, newest first; `enabled` is false when the workspace has no storage for them + content: + application/json: + schema: + type: object + required: + - enabled + - sessions + properties: + enabled: + type: boolean + storage_id: + type: string + description: names the storage answered from; sync state recorded against another one is void + backup_generation: + type: integer + description: bumped by every workspace key rotation; sync state recorded under another one is void + fallback: + type: boolean + description: the storage answered from is the instance object store, standing in for a workspace without storage of its own; a removal owed to it is retired by any answer from the workspace's own storage + sessions: + type: array + description: the newest 500 at most + items: + $ref: "#/components/schemas/AISessionBackupListing" + truncated: + type: boolean + description: the user has more sessions than the answer names + + /w/{workspace}/ai/sessions/pull: + post: + summary: fetch whole AI session backups + operationId: pullAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - ids + properties: + ids: + type: array + maxItems: 20 + items: + type: string + resume: + $ref: "#/components/schemas/AISessionBackupCursor" + responses: + "200": + description: the backups found; `deferred` lists ids that did not fit the response budget + content: + application/json: + schema: + type: object + required: + - enabled + - sessions + - deferred + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + fallback: + type: boolean + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackup" + deferred: + type: array + items: + type: string + + /w/{workspace}/ai/sessions/push: + post: + summary: write changed pieces of AI sessions to their backups, and remove deleted ones + operationId: pushAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - owner + properties: + owner: + type: string + description: the email the push was prepared for; refused with a 409 when it is not the caller's + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackupPush" + removed: + type: array + items: + type: string + responses: + "200": + description: one result per session written or removed, in request order + content: + application/json: + schema: + type: object + required: + - enabled + - results + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + fallback: + type: boolean + results: + type: array + items: + type: object + required: + - id + properties: + id: + type: string + error: + type: string + needs_whole: + type: boolean + description: nothing was written and the session must be pushed whole again; an incremental part found no listed session to ride on (the backup was removed, or a push split over parts is in progress or was abandoned), or a later part of a push split over parts found another push had superseded it + + /w/{workspace}/ai/shared_artifacts/share: + post: + summary: share an AI session artifact with the workspace + description: > + Stores a read-only copy that any member of the workspace can open by id. Sharing the + same artifact again updates that copy, keeps its id, and restarts its retention window. + operationId: shareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - artifact_id + - name + - kind + - version + - content + properties: + artifact_id: + type: string + description: the artifact's id in the author's session + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + minimum: 1 + content: + type: string + responses: + "200": + description: the shared copy + content: + application/json: + schema: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/status: + get: + summary: get the calling user's share of one of their AI session artifacts + operationId: getAiArtifactShareStatus + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: artifact_id + in: query + required: true + schema: + type: string + responses: + "200": + description: the live share, if any, and how long shares last + content: + application/json: + schema: + type: object + required: + - retention_secs + properties: + retention_secs: + type: integer + share: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/get/{id}: + get: + summary: get a shared AI session artifact + operationId: getSharedAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: the shared artifact + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/SharedAiArtifactInfo" + - type: object + required: + - content + - can_unshare + properties: + content: + type: string + can_unshare: + type: boolean + description: whether the caller authored the share or is a workspace admin + + /w/{workspace}/ai/shared_artifacts/delete/{id}: + delete: + summary: stop sharing an AI session artifact + operationId: unshareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: share deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by @@ -13126,8 +13966,9 @@ paths: Unauthenticated: what a signed-out visitor reads to learn that signing in would let them in. 404 unless the app's execution mode is `guest` AND the workspace has `guest_access_enabled` AND the instance has not set the - `guest_access_disabled` global setting, so it says nothing about apps that - are not open to guests. Discloses only the app path, to a caller already + `guest_access_disabled` global setting, and never on a deployment where guests + are unavailable (the shared cloud), so it says nothing about apps that are not + open to guests. Discloses only the app path, to a caller already holding the share secret. operationId: getGuestEntry tags: @@ -16641,6 +17482,7 @@ paths: - can_approve - user_auth_required - approvers + - skin properties: flow_id: type: string @@ -16673,6 +17515,16 @@ paths: hide_cancel: type: boolean description: whether to hide the cancel button in the UI + skin: + type: string + enum: [detailed, minimal] + description: how the approval page presents the request + step_summary: + type: string + description: summary of the approval step, for the page title + flow_summary: + type: string + description: summary of the flow or workflow the approval belongs to approvers: type: array items: @@ -18746,6 +19598,49 @@ paths: schema: type: string + /w/{workspace}/native_triggers/{service_name}/setenabled/{external_id}: + post: + summary: set enabled state of native trigger + description: | + Enables or disables a native trigger. A disabled trigger stays registered on the + external service but starts no job when it fires. + Requires write access to the script or flow that the trigger is associated with. + operationId: setNativeTriggerEnabled + tags: + - native_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: service_name + in: path + required: true + schema: + $ref: "#/components/schemas/NativeServiceName" + - name: external_id + in: path + required: true + description: The external ID of the trigger from the external service + schema: + type: string + requestBody: + description: updated enabled state + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + "200": + description: native trigger enabled state updated + content: + text/plain: + schema: + type: string + /w/{workspace}/native_triggers/{service_name}/list: get: summary: list native triggers @@ -21581,6 +22476,98 @@ paths: - id - values + /workers/queue_metrics_series: + get: + summary: get the queue metrics of a time window, as a bounded line per tag + operationId: getQueueMetricsSeries + tags: + - worker + parameters: + - name: window_secs + in: query + required: false + description: how far back to read, in seconds (defaults to one day, capped at the 14-day retention) + schema: + type: integer + responses: + "200": + description: jobs waiting and queue delay per tag, as the vertices of lines joined by straight segments + content: + application/json: + schema: + type: object + properties: + from: + type: integer + description: start of the window, in epoch milliseconds + to: + type: integer + description: end of the window, in epoch milliseconds + tags: + type: array + items: + type: object + properties: + tag: + type: string + count: + type: array + description: "[epoch ms, jobs waiting more than 3 seconds] vertices" + items: + type: array + items: + type: number + delay: + type: array + description: "[epoch ms, seconds the next job has waited] vertices" + items: + type: array + items: + type: number + required: + - tag + - count + - delay + required: + - from + - to + - tags + + /workers/queue_status: + get: + summary: get the live queue status of every tag with jobs waiting or running + operationId: getQueueStatus + tags: + - worker + responses: + "200": + description: queue status per tag + content: + application/json: + schema: + type: array + items: + type: object + properties: + tag: + type: string + waiting: + type: integer + description: jobs due for more than 3 seconds that no worker has picked up + delay: + type: number + description: seconds the job the next pull would take has been waiting, absent when none is + running: + type: integer + workers: + type: integer + description: workers that pinged in the last minute and pull this tag + required: + - tag + - waiting + - running + - workers + /workers/queue_counts: get: summary: get counts of jobs waiting for an executor per tag @@ -24364,6 +25351,72 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/assets/column_lineage: + get: + summary: Column-level lineage of a set of dbt relations + description: > + The direct (`copy` / `mod`) column-to-column lineage the given relations' + columns sit in — the connected component around them, from the engine's + static analysis. Not their own edges, which would stop one hop out since + a column trace walks transitively, and not a whole project's, which + carries model families the selection cannot reach. + + Several relations, answered as one union, because one selection reaches + several: a script's output column can derive from columns of several dbt + models. Unpinned, the component crosses projects — a relation one project + produces is another's source — and the caller's access is decided again + for every project it reaches, so a trace ends where their grants do. A + pinned answer, by version here or by job on the run route, is one + project's. + + Its own endpoint rather than a field on the asset graph: the graph is + folder-wide and polled by a run page, while this is rendered for one + selection at a time. Empty for projects that did not opt into the + analysis pass (`column_lineage: true`), which is the ordinary case. The + indirect `scan` kind is stored but never served: it reaches every output + column of its model. + operationId: getDbtColumnLineage + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. + schema: + type: array + maxItems: 1000 + minItems: 1 + items: + type: string + - name: dbt_script_hash + in: query + description: > + The deployed version a view is drawing, when it is drawing one — the + dbt editor, which shows a single project as of a single deploy. A + version-pinned answer is that version's project alone, the same as a + job-pinned one, and only the unpinned answer crosses projects: a pin + says which stored graph is on screen, and another project's live + graph is not part of it. + + A run's or an editor buffer's own graph is not reachable here: that + pins to a job, and costs the job-read gate — see + `jobs/dbt_column_lineage/{id}`. + schema: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/assets/macros: get: summary: List every workspace DuckDB macro (deployed `// macros` libraries) @@ -24555,6 +25608,53 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/jobs/dbt_column_lineage/{id}: + get: + summary: Get relations' project column lineage as one run saw it + description: > + The same answer as `assets/column_lineage`, for the project version a + single job ran — including the dbt editor's parse of its own buffer, + whose graph belongs to that job and is reachable no other way. One + project answers here, the one the run is of, since the graph this + annotates is that project's too. Authorized through the job, the same + gate as `dbt_graph`. Reaching the run is not on its own enough to read + the project: a caller with no access to the script gets its relations and + `ref()` edges from `dbt_graph` and an empty answer here, exactly as that + endpoint redacts the model's SQL. + operationId: getDbtRunColumnLineage + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: The job whose graph the lineage is read from + schema: + type: string + format: uuid + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. + schema: + type: array + maxItems: 1000 + minItems: 1 + items: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/jobs/run_progress/{id}: get: summary: List the per-relation progress one job has recorded so far @@ -25431,6 +26531,30 @@ paths: schema: type: string + /w/{workspace}/hub/projects: + get: + summary: list the hub's published projects + description: | + Forwards to the configured Hub's public project catalogue and returns its + status code and raw response body. Readable by any workspace member: the + listing is not workspace-scoped, and it is proxied only because the Hub's + listing endpoint sends no CORS header. Refused with 400 when the instance + has the Hub disabled, in which case no outbound request is made. + operationId: listHubProjects + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + "400": + description: the Hub is disabled on this instance + /w/{workspace}/hub/project: get: summary: get the hub project linked to a workspace folder @@ -26148,6 +27272,40 @@ components: drawn identically and the ambiguity would otherwise just move into the editor. Omitted for the unpinned workspace graph, which spans every project and so has no one time. + DbtColumnLineage: + type: object + description: >- + The direct column-to-column lineage the asked-for relations' columns sit + in — the connected component around them — in the terms the canvas draws: + relations and columns, never dbt's node ids. + required: [edges, truncated] + properties: + edges: + type: array + items: + type: object + required: [from_asset_path, from_column, to_asset_path, to_column, kind] + properties: + from_asset_path: + type: string + from_column: + type: string + to_asset_path: + type: string + to_column: + type: string + kind: + type: string + description: >- + dbt's own word for how the value travelled — `copy` + (passthrough) or `mod` (transformed). Not an enum: the engine + treats the set as open. + truncated: + type: boolean + description: >- + The component reaches further than `edges`, which holds the part + nearest the asked-for relations. A trace that stops short is + otherwise indistinguishable from one that ends. DbtAssetProvenance: type: object description: >- @@ -26195,7 +27353,23 @@ components: columns: type: object additionalProperties: true - description: Declared column metadata (name -> description). NOT column lineage — `manifest.json` carries none. + description: Declared column metadata (name -> description) — what `manifest.json` carries, which is only the columns an author wrote down. Omitted when the caller cannot read the script. + column_schema: + type: array + description: >- + Every column of the relation, typed and in the order the model + produces them, from the engine's static analysis. Present only for a + project that opted into it, and gated like `columns` and the model's + SQL: a full column list is the shape of what the author wrote. + items: + type: object + required: [name] + properties: + name: + type: string + type: + type: string + description: The declared type where `schema.yml` gives one, else the inferred one. Omitted when neither is known. freshness: type: object additionalProperties: true @@ -27319,6 +28493,163 @@ components: fixes) from the workspace UI. Read from the workspace's own settings even when the providers served fall back to the instance config. AI agent steps and the AI sandbox in flows are unaffected. + sessions_storage_disabled: + type: boolean + description: >- + Stops browsers from backing their AI sessions up to the workspace's object + storage. Read from the workspace's own settings like `copilot_disabled`. + sessions_retention_days: + type: integer + minimum: 1 + maximum: 3650 + description: >- + The server deletes the backup of a session no push has reached for this many + days. Unset keeps backups until the user deletes the session. Read from the + workspace's own settings like `copilot_disabled`. + + AISessionBackupListing: + type: object + required: + - id + - updated_at + - epoch + properties: + id: + type: string + updated_at: + type: string + format: date-time + epoch: + type: integer + description: the session's move count when this copy was pushed; of a session two workspaces list, the copy with the higher one is the later + + AISessionBackupImage: + type: object + required: + - chat_id + - id + - data_url + properties: + chat_id: + type: string + id: + type: string + data_url: + type: string + + AISessionBackupChat: + type: object + required: + - id + - record + properties: + id: + type: string + record: + type: object + additionalProperties: true + + AISessionBackup: + type: object + required: + - id + - head + - chats + - images + - listing + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + next: + $ref: "#/components/schemas/AISessionBackupCursor" + listing: + type: string + description: a fingerprint of the session's listing; pages of one session whose fingerprints differ do not belong together + moved: + type: boolean + description: the backup kept changing while this page was read, so it may mix two versions; the browser starts the session over + + AISessionBackupCursor: + type: object + description: where a pull of a session that did not fit one answer whole picks up; the rest of the session follows a pull naming that session alone with this as `resume` + required: + - id + - images + - after + properties: + id: + type: string + images: + type: boolean + after: + type: string + + AISessionBackupPush: + type: object + required: + - id + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + delete_chats: + type: array + items: + type: string + delete_images: + type: array + items: + type: object + required: + - chat_id + - id + properties: + chat_id: + type: string + id: + type: string + partial: + type: boolean + description: more parts of this session follow, in this push or a later one; the session is not listed on this one. Such a part names its push (`push`), or it is refused + whole: + type: boolean + description: a part of a push of the session whole; the head is on the part that opens it, which replaces whatever the storage holds of the session, and every piece the browser has is on one of them. An incremental part instead rides on a session the storage lists and is refused with needs_whole when it lists none + push: + type: string + description: a push split over several parts names itself on each with a token the browser draws; the part that opens it unlists the session and the last part lists it again, and a later part is written only while that token is the one there (refused with needs_whole otherwise) + opens: + type: boolean + description: this part opens the push named by `push` + epoch: + type: integer + description: the session's move count (its record's `moves`), kept with the marker that lists the session; an incremental part rides on the marker of the same count FreeTierInfo: type: object @@ -27418,6 +28749,36 @@ components: - output_tokens - requests + SharedAiArtifactInfo: + type: object + properties: + id: + type: string + format: uuid + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + created_by: + type: string + shared_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + required: + - id + - name + - kind + - version + - created_by + - shared_at + - expires_at + InstanceAIProviderSummary: type: object properties: @@ -28912,7 +30273,12 @@ components: to one seat: `billable_guests`, `guest_seats`); every other plan and build admits no new email until the count drops. `instance_enabled` is the superadmin switch (`guest_access_disabled` global setting) every workspace switch sits under. + `available` is whether this deployment can have guests at all: false on the shared + cloud, where guest access requires a self-hosted or dedicated deployment, and every + other field and switch is then moot. properties: + available: + type: boolean instance_enabled: type: boolean guest_count: @@ -28932,6 +30298,7 @@ components: type: integer format: int64 required: + - available - instance_enabled - guest_count - window_days @@ -29273,6 +30640,51 @@ components: - operation - action_kind + TrashItem: + type: object + properties: + id: + type: integer + format: int64 + workspace_id: + type: string + item_kind: + type: string + description: script, flow, app, schedule, variable, resource, or a trigger kind such as http_trigger + item_path: + type: string + deleted_by: + type: string + deleted_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + description: when the item is permanently deleted unless restored first + required: + - id + - workspace_id + - item_kind + - item_path + - deleted_by + - deleted_at + - expires_at + + TrashItemWithData: + allOf: + - $ref: "#/components/schemas/TrashItem" + - type: object + properties: + item_data: + type: object + additionalProperties: true + description: > + the deleted rows as they were stored; the shape depends on the kind, and a + secret variable's value stays encrypted + required: + - item_data + MainArgSignature: type: object properties: @@ -29730,6 +31142,11 @@ components: type: string is_fileset: type: boolean + display_name: + type: string + description: >- + The name the product goes by, e.g. "Google Sheets" for gsheets. + Absent where nobody named the type. required: - name @@ -29747,6 +31164,12 @@ components: description: >- File extension for a type whose value is one file rather than a set of fields. Omit to leave it unchanged; send null to clear it. + display_name: + type: string + nullable: true + description: >- + The name the product goes by. Omit to leave it unchanged; send null + to clear it. TriggerHistoryEntry: type: object @@ -30450,6 +31873,14 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare @@ -30541,6 +31972,14 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare @@ -30639,6 +32078,14 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare @@ -33129,7 +34576,7 @@ components: type: string login_type: type: string - enum: ["password", "github", "service_account"] + enum: ["password", "github", "service_account", "pending_oauth"] super_admin: type: boolean devops: @@ -33397,8 +34844,10 @@ components: checked when the session is minted and again on every guest request on_behalf_of: type: string + description: The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity. on_behalf_of_email: type: string + description: Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected. sandbox: type: boolean description: > @@ -33419,7 +34868,8 @@ components: carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, - users:read, resources:read, variables:read). + users:read, resources:read, variables:read, flow_conversations:read, + flow_conversations:write). ListableApp: type: object @@ -34240,9 +35690,58 @@ components: open_pr_error: type: string description: server-owned, last failure opening a PR for a deploy branch of this repo + credential: + $ref: "#/components/schemas/GitCredentialStatus" required: - git_repo_resource_path + GitCredentialStatus: + type: object + description: server-owned, what the repo's own credential reports about itself + properties: + provider: + type: string + enum: + - gitlab + token_id: + type: integer + format: int64 + expires_at: + type: string + format: date + description: absent for a non-expiring token + scopes: + type: array + items: + type: string + rotatable: + type: boolean + description: whether this workspace renews the credential itself + checked_at: + type: integer + format: int64 + error: + type: string + required: + - provider + - rotatable + - checked_at + + GitlabProject: + type: object + description: a GitLab project a token can sync, as the resource form needs it + properties: + path_with_namespace: + type: string + description: nested group path plus project name, which is also GitLab's project id + http_url_to_repo: + type: string + default_branch: + type: string + required: + - path_with_namespace + - http_url_to_repo + AutoPullMode: type: string enum: @@ -34295,6 +35794,9 @@ components: type: string last_pull_status: $ref: "#/components/schemas/AutoPullStatus" + enabled_by: + type: string + description: Email of the admin automatic pulls apply changes as. Set by the server when the settings are saved. required: - enabled @@ -35430,6 +36932,9 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: Whether the trigger starts a job when it fires required: - external_id - workspace_id @@ -35437,6 +36942,7 @@ components: - script_path - is_flow - service_config + - enabled NativeTriggerWithExternal: type: object @@ -35468,6 +36974,9 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: Whether the trigger starts a job when it fires external_data: type: object nullable: true @@ -35488,6 +36997,7 @@ components: - script_path - is_flow - service_config + - enabled - external_data WorkspaceIntegrations: @@ -35573,6 +37083,12 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: >- + Whether the trigger starts a job when it fires. Honoured on create only, so a + trigger can be registered already paused; an update ignores it and setenabled is + the only way to change an existing trigger's state. Defaults to true. required: - script_path - is_flow diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index e70cc19544..1710db9cec 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -451,6 +451,16 @@ pub struct AIConfig { /// and the AI sandbox are unaffected, so the providers stay in force. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub copilot_disabled: bool, + /// Stops browsers from backing their AI sessions up to the workspace's object storage + /// (`ai_sessions.rs`). Read from the workspace's own row like `copilot_disabled`. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub sessions_storage_disabled: bool, + /// The server's sweep (`ai_sessions.rs`) deletes the backup of a session no push has + /// reached for this many days. The copies in members' browsers are untouched. Unset + /// keeps backups until the user deletes the session. Read from the workspace's own row + /// like `copilot_disabled`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sessions_retention_days: Option, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the @@ -487,6 +497,9 @@ impl ModelPriceOverride { } } +/// Ten years: past any plausible retention, and well within what a day count is turned into. +pub const MAX_SESSIONS_RETENTION_DAYS: u32 = 3650; + impl AIConfig { pub fn validate_model_pricing(&self) -> Result<()> { for (key, price) in self.model_pricing.iter().flatten() { @@ -495,6 +508,17 @@ impl AIConfig { Ok(()) } + pub fn validate_sessions_retention(&self) -> Result<()> { + match self.sessions_retention_days { + Some(days) if !(1..=MAX_SESSIONS_RETENTION_DAYS).contains(&days) => { + Err(Error::BadRequest(format!( + "AI session retention must be between 1 and {MAX_SESSIONS_RETENTION_DAYS} days (got {days})" + ))) + } + _ => Ok(()), + } + } + pub fn has_providers(&self) -> bool { self.providers .as_ref() @@ -518,11 +542,18 @@ pub fn workspaced_service() -> Router { // could make the server allocate and parse an arbitrarily large one. // Sized well above a full batch of the shape below. .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ) + .nest( + "/shared_artifacts", + crate::ai_shared_artifacts::workspaced_service(), ); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); + #[cfg(feature = "parquet")] + let router = router.nest("/sessions", crate::ai_sessions::workspaced_service()); + router } diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs new file mode 100644 index 0000000000..65a8137c4e --- /dev/null +++ b/backend/windmill-api/src/ai_sessions.rs @@ -0,0 +1,1634 @@ +//! Lazily replicated backups of the browser's AI sessions in the workspace's object storage. +//! +//! The browser keeps the sessions in IndexedDB and pushes changed pieces here in batches; an +//! empty browser restores from what was pushed. The server owns the key layout, keeps the +//! caller's own prefix the only one it can reach, and encrypts every object with the +//! workspace key so bucket credentials do not read transcripts: +//! +//! ```text +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/head.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/chats/{cid}.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/artifacts.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/images/{sid}/{cid}/{iid} +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/index/{sid}/{epoch} +//! ``` +//! +//! The index marker is empty, written last by every push of the session, and is what a +//! listing reads: one object per session, whatever the session holds, its `last_modified` +//! the session's `updated_at`. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Path}, + routing::{get, post}, + Extension, Json, Router, +}; +use futures::{StreamExt, TryStreamExt}; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::sync::Arc; +use windmill_api_auth::is_effectively_unscoped; +use windmill_api_workspaces::ai_session_backups::{ + fallback_store, generation_prefix, primary_store, storage_id, MAX_OBJECT_BYTES, +}; +use windmill_api_workspaces::workspaces::sessions_retention_days; +use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload, +}; +use windmill_object_store::{build_object_store_client, object_store_error_to_error}; + +const PUSH_BODY_LIMIT: usize = MAX_OBJECT_BYTES; +/// A pull names at most MAX_PULL_IDS ids of 64 bytes; anything larger is not a pull. +const PULL_BODY_LIMIT: usize = 64 * 1024; +/// A pull answer larger than this hands the remaining ids back as `deferred`. +const PULL_RESPONSE_BUDGET: usize = 32 * 1024 * 1024; +const MAX_HEAD_BYTES: usize = 1024 * 1024; +/// What the cipher adds to a plaintext at most (a block of padding): an object stored at a +/// cap is that much larger than the cap when read back. +const CIPHER_PADDING: usize = 16; +/// The browser bounds an image to a 1568 px edge and re-encodes past 700 KB; this is +/// well above what that produces. +const MAX_IMAGE_BYTES: usize = 4 * 1024 * 1024; +const MAX_PULL_IDS: usize = 20; +const MAX_PUSH_SESSIONS: usize = 100; +const MAX_REMOVED: usize = 200; +const MAX_CHATS_PER_ENTRY: usize = 100; +const MAX_IMAGES_PER_ENTRY: usize = 500; +const MAX_DELETES_PER_ENTRY: usize = 1000; +const MAX_OPERATIONS_PER_PUSH: usize = 4000; +/// Entries of listing metadata a pull holds per page of a session. +const MAX_LISTED_OBJECTS: usize = 5000; +/// Session markers a listing scans, and the newest sessions it answers with. +const MAX_LIST_SCAN: usize = 50_000; +const LIST_MAX: usize = 500; +const IO_CONCURRENCY: usize = 8; +/// Sessions the retention sweep deletes per workspace and pass at most; the rest wait for +/// the next pass. +const SWEEP_MAX_PER_WORKSPACE: usize = 1000; +/// Session-level advisory lock of the retention sweep: one server at a time runs it. +const SWEEP_LOCK_ID: i64 = 0x5745_4550_4149; +/// The name of the sweep's record next to a session's markers (see `Backend::sweep_key`). +const SWEEP_RECORD: &str = "sweep"; +/// The name of a split push's token next to a session's markers (see `Backend::push_key`). +const PUSH_TOKEN: &str = "push"; + +/// A marker modified before this is past a retention of `days`. +fn retention_cutoff(days: u32) -> chrono::DateTime { + chrono::Utc::now() - chrono::Duration::days(i64::from(days)) +} + +/// What a key under the `index/` prefix is. +enum IndexEntry { + /// The marker that lists the session, named by its epoch. + Marker(u32), + /// The retention sweep's record (see `Backend::sweep_key`). + Sweep, + /// The token of a push split over parts (see `Backend::push_key`). + Push, +} + +/// The session a key under the `index/` prefix belongs to, and what the key is. +fn index_entry<'a>(index: &ObjectPath, key: &'a ObjectPath) -> Option<(&'a str, IndexEntry)> { + // `Path` drops the trailing delimiter, so the remainder starts with one. + let rel = key.as_ref().strip_prefix(index.as_ref())?; + let (sid, name) = rel.trim_start_matches('/').split_once('/')?; + if sid.is_empty() { + return None; + } + let entry = match name { + SWEEP_RECORD => IndexEntry::Sweep, + PUSH_TOKEN => IndexEntry::Push, + epoch => IndexEntry::Marker(epoch.parse().ok()?), + }; + Some((sid, entry)) +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list)) + .route( + "/pull", + post(pull).layer(DefaultBodyLimit::max(PULL_BODY_LIMIT)), + ) + .route( + "/push", + post(push).layer(DefaultBodyLimit::max(PUSH_BODY_LIMIT)), + ) +} + +/// What reading an object yields. `Gone`: not there (deleted since the listing, or never +/// pushed). `Grown`: larger than expected, so replaced since the listing (or planted), and +/// left unread; a pull answers with a page ending before it rather than without it. +/// `Foreign`: it does not decrypt for this user (written under another user's or +/// workspace's key), and must not take the rest of the session down with it. +enum Read { + Text(String), + Gone, + Grown, + Foreign, +} + +/// The user's prefix in the workspace storage, plus what reads and writes it. +struct Backend { + store: Arc, + mc: MagicCrypt256, + prefix: String, + /// Name the storage and the generation the objects are under, for the browser's sync + /// state: a row recorded against another storage or generation is stale, a removal is + /// owed to the storage alone (a rotation deleted the older generation's copy anyway). + storage_id: String, + generation: i64, + /// `ai_config.sessions_retention_days`: a session whose marker is older is not listed, + /// whether or not the sweep has deleted it yet. + retention_days: Option, + /// The store is the instance object store standing in for a workspace without storage + /// of its own (`ai_session_backups::fallback_store`). + fallback: bool, +} + +impl Backend { + fn index_prefix(&self) -> ObjectPath { + ObjectPath::from(format!("{}/index/", self.prefix)) + } + + /// The moment a marker's modification time must reach to count as live, under the + /// workspace's retention; `None` without one. + fn retention_cutoff(&self) -> Option> { + self.retention_days.map(retention_cutoff) + } + + /// The marker that lists the session, named by the session's move count so that of a + /// session two workspaces list, the copy moved last is told from the listing alone. + fn index_key(&self, sid: &str, epoch: u32) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{epoch}", self.prefix)) + } + + fn index_session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/", self.prefix)) + } + + /// Written by the retention sweep before it deletes anything of a session, and deleted + /// last (`remove_session`): what finds a removal the sweep started and could not finish, + /// the markers being gone by then. Not an epoch, so nothing lists or pulls a session by it. + fn sweep_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{SWEEP_RECORD}", self.prefix)) + } + + /// The token of the push split over parts in progress, next to the markers so a removal + /// or the next whole push clears it with them, and the retention sweep, which walks the + /// markers, finds one a browser abandoned. + fn push_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{PUSH_TOKEN}", self.prefix)) + } + + fn session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/", self.prefix)) + } + + fn head_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/head.json", self.prefix)) + } + + fn chat_key(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/chats/{cid}.json", self.prefix)) + } + + fn artifacts_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/artifacts.json", self.prefix)) + } + + fn images_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/", self.prefix)) + } + + fn chat_images_prefix(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/", self.prefix)) + } + + fn image_key(&self, sid: &str, cid: &str, iid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/{iid}", self.prefix)) + } + + fn seal(&self, plaintext: &[u8]) -> Vec { + self.mc.encrypt_bytes_to_bytes(plaintext) + } + + /// Bytes written. + async fn put_sealed(&self, key: &ObjectPath, ciphertext: Vec) -> Result { + let written = ciphertext.len(); + self.store + .put(key, PutPayload::from(ciphertext)) + .await + .map_err(object_store_error_to_error)?; + Ok(written) + } + + async fn put(&self, key: &ObjectPath, plaintext: &[u8]) -> Result { + self.put_sealed(key, self.seal(plaintext)).await + } + + /// `max` is what the listing said the object holds, or the cap of its kind for one read + /// without a listing: checked before buffering, since whoever holds the bucket's + /// credentials can put anything at a predictable key. + async fn get(&self, key: &ObjectPath, max: usize) -> Result { + let result = match self.store.get(key).await { + Ok(result) => result, + Err(ObjectStoreError::NotFound { .. }) => return Ok(Read::Gone), + Err(e) => return Err(object_store_error_to_error(e)), + }; + let size = result.meta.size as usize; + // Larger than any push writes: planted, whatever the listing said, and skipped like + // an object of another key rather than retried like one that grew. + if size > MAX_OBJECT_BYTES { + tracing::warn!("AI session backup object {key} is larger than any push writes"); + return Ok(Read::Foreign); + } + if size > max { + return Ok(Read::Grown); + } + let bytes = result.bytes().await.map_err(object_store_error_to_error)?; + // The objects are JSON and data URLs: a wrong key's output failing UTF-8 tells it + // apart beyond the cipher's padding check, which a wrong key passes now and then. + match self + .mc + .decrypt_bytes_to_bytes(&bytes) + .ok() + .and_then(|plaintext| String::from_utf8(plaintext).ok()) + { + Some(text) => Ok(Read::Text(text)), + None => { + tracing::warn!("AI session backup object {key} does not decrypt for its reader"); + Ok(Read::Foreign) + } + } + } + + async fn delete(&self, key: &ObjectPath) -> Result<()> { + match self.store.delete(key).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// The entries under `prefix` past `after` in key order, as many as fit `budget` bytes; + /// `true` when more follow. Every key past `after` is seen and the MAX_LISTED_OBJECTS + /// smallest kept (a max-heap dropping its largest), since a page is defined by key + /// order and the store promises none; that cap is what bounds a pull's memory, a + /// session growing by valid pushes without limit. With `at_least_one`, the first entry + /// is taken whatever its size, so an answer owed the session makes progress on it (no + /// object exceeds the push body cap). + async fn list_within( + &self, + prefix: &ObjectPath, + after: Option<&ObjectPath>, + budget: usize, + at_least_one: bool, + ) -> Result<(Vec<(ObjectPath, usize)>, bool)> { + let mut kept: std::collections::BinaryHeap<(ObjectPath, usize)> = Default::default(); + let mut dropped = false; + let mut stream = match after { + Some(after) => self.store.list_with_offset(Some(prefix), after), + None => self.store.list(Some(prefix)), + }; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + kept.push((meta.location, meta.size as usize)); + if kept.len() > MAX_LISTED_OBJECTS { + kept.pop(); + dropped = true; + } + } + let mut entries = vec![]; + let mut total = 0; + for (key, size) in kept.into_sorted_vec() { + if total + size > budget && !(at_least_one && entries.is_empty()) { + return Ok((entries, true)); + } + total += size; + entries.push((key, size)); + } + Ok((entries, dropped)) + } + + /// Bytes written. Sealed up front so every stream item is owned: an item borrowing + /// from the request makes the future higher-ranked over that lifetime, which the + /// handler's `Send` bound cannot prove. + async fn put_all(&self, puts: Vec<(ObjectPath, Vec)>) -> Result { + futures::stream::iter(puts) + .map(|(key, ciphertext)| async move { self.put_sealed(&key, ciphertext).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_fold(0, |acc, n| async move { Ok::<_, Error>(acc + n) }) + .await + } + + async fn delete_all(&self, keys: Vec) -> Result<()> { + futures::stream::iter(keys) + .map(|key| async move { self.delete(&key).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_collect::>() + .await?; + Ok(()) + } + + /// A fingerprint of the session's marker and of everything listed under its two + /// prefixes (key, size, modification time, entity tag and version), combined as the + /// listing streams and in no particular order, so a session of any size costs bounded + /// memory. `None` for a session + /// the storage does not list. Taken before and after a page is read, so a page a push + /// changed under is read again; pages of one pull carry it, and the browser starts the + /// session over when it moved between two of them. + async fn listing_fingerprint(&self, sid: &str) -> Result> { + use std::hash::{DefaultHasher, Hash, Hasher}; + // The entity tag and version go in with the key, size and time: a store reports + // modification times coarsely, and an object rewritten at the same size within that + // grain would otherwise fingerprint the same. + fn fold( + acc: u64, + location: &str, + size: S, + modified: i64, + e_tag: Option<&str>, + version: Option<&str>, + ) -> u64 { + let mut hasher = DefaultHasher::new(); + (location, size, modified, e_tag, version).hash(&mut hasher); + acc.wrapping_add(hasher.finish()) + } + let mut acc = 0u64; + let mut listed = false; + for (marker, prefix) in [ + (true, self.index_session_prefix(sid)), + (false, self.session_prefix(sid)), + (false, self.images_prefix(sid)), + ] { + let mut stream = self.store.list(Some(&prefix)); + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + // The sweep's record is not a marker: a session it started removing is absent. + listed |= marker + && meta + .location + .filename() + .is_some_and(|name| name.parse::().is_ok()); + acc = fold( + acc, + meta.location.as_ref(), + meta.size, + meta.last_modified.timestamp_millis(), + meta.e_tag.as_deref(), + meta.version.as_deref(), + ); + } + } + if !listed { + return Ok(None); + } + Ok(Some(format!("{acc:016x}"))) + } + + async fn exists(&self, key: &ObjectPath) -> Result { + match self.store.head(key).await { + Ok(_) => Ok(true), + Err(ObjectStoreError::NotFound { .. }) => Ok(false), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// Deletes as the listing streams, so a prefix of any size costs bounded memory. + async fn delete_prefix(&self, prefix: &ObjectPath) -> Result<()> { + self.store + .list(Some(prefix)) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + self.delete(&meta.location).await + }) + .await + } +} + +/// Backups are the user's own browser state and nothing else may reach them: a job token +/// may carry an `on_behalf_of` identity, and every scoped token (guest, embed, app policy, +/// MCP) is minted for something narrower than the user's whole assistant history. +fn require_plain_user_token(authed: &ApiAuthed) -> Result<()> { + if authed.job_id.is_some() || !is_effectively_unscoped(authed.scopes.as_deref()) { + return Err(Error::PermissionDenied( + "AI session backups are only reachable with an unscoped user token".to_string(), + )); + } + Ok(()) +} + +/// Every key is assembled server-side from ids the browser mints (`createLongHash` and +/// `randomUUID` forms), so anything outside this alphabet is a forged id, not a real one. +fn require_valid_id(kind: &str, id: &str) -> Result<()> { + let ok = !id.is_empty() + && id.len() <= 64 + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if !ok { + return Err(Error::BadRequest(format!("invalid {kind} id: {id:?}"))); + } + Ok(()) +} + +/// Images travel as base64 data URLs and are stored verbatim, so they serialize back into +/// a pull answer at exactly their stored size; anything else (control characters, +/// quotes) could grow several times under JSON escaping and defeat the pull budget. +fn require_data_url(data_url: &str) -> Result<()> { + let ok = data_url.len() <= MAX_IMAGE_BYTES + && data_url + .strip_prefix("data:") + .and_then(|rest| rest.split_once(";base64,")) + .is_some_and(|(mime, payload)| { + !mime.is_empty() + && mime.bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'/' | b'.' | b'+' | b'-') + }) + && payload + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')) + }); + if !ok { + return Err(Error::BadRequest( + "an image must be a base64 data URL within the size cap".to_string(), + )); + } + Ok(()) +} + +fn require_json_object(kind: &str, raw: &RawValue, max_bytes: usize) -> Result<()> { + let text = raw.get(); + if !text.trim_start().starts_with('{') { + return Err(Error::BadRequest(format!("{kind} must be a JSON object"))); + } + if text.len() > max_bytes { + return Err(Error::BadRequest(format!( + "{kind} exceeds {max_bytes} bytes" + ))); + } + Ok(()) +} + +/// `None` when the workspace has nowhere to keep backups: no primary storage configured and +/// no instance store to stand in, or the admin switched them off. Both read as +/// `enabled: false` so the browser stops trying. +async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result> { + let (disabled, retention, generation, has_storage) = + sqlx::query_as::<_, (Option, Option, i64, bool)>( + "SELECT (ai_config->>'sessions_storage_disabled')::bool, \ + ai_config->'sessions_retention_days', ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL \ + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_optional(db) + .await? + .unwrap_or((None, None, 0, false)); + if disabled.unwrap_or(false) { + return Ok(None); + } + let retention_days = sessions_retention_days(retention.as_ref()); + // Decided from the row the generation came from: the instance store is written only + // under a generation read while the workspace had no storage of its own, which + // configuring one moves past (`ai_session_backups`). + let (store, storage_id, fallback) = if has_storage { + let (_, resource) = + crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; + let Some(resource) = resource else { + return Ok(None); + }; + ( + build_object_store_client(&resource).await?, + storage_id(&resource), + false, + ) + } else { + // The instance store stands in, under the same layout and the same key. + match fallback_store(db).await? { + Some(f) => (f.store, f.storage_id, true), + None => return Ok(None), + } + }; + let user = calculate_hash(&authed.email); + // Keyed per user, not per workspace: anyone who can write the bucket could otherwise copy + // another member's ciphertext under their own prefix and have `pull` decrypt it for them. + let key = get_workspace_key(w_id, db).await?; + let mc = crypt_from_key_with_suffix(&key, &user); + let prefix = format!("{}/{user}", generation_prefix(w_id, generation)); + Ok(Some(Backend { + store, + mc, + prefix, + storage_id, + generation, + retention_days, + fallback, + })) +} + +#[derive(Serialize)] +struct SessionListing { + id: String, + updated_at: chrono::DateTime, + /// The session's move count when this copy was pushed (see `PushedSession::epoch`). + epoch: u32, +} + +#[derive(Serialize)] +struct ListResponse { + enabled: bool, + /// The storage answered from, and the generation a key rotation bumps; a browser whose + /// sync state names another storage or generation starts over. + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + /// The storage is the instance store standing in for a workspace without one of its + /// own; a removal owed to it is retired by any answer from the workspace's own storage + /// once it has one (configuring it moved the generation past everything the workspace + /// left in any instance store). + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, + sessions: Vec, + /// The user has more sessions than the answer names. + #[serde(skip_serializing_if = "std::ops::Not::not")] + truncated: bool, +} + +/// A session is listed once a push entry of it landed whole (its marker is written last); +/// a push that failed before that left objects the listing does not name. One past the +/// workspace's retention is not listed either, whether or not the sweep has reached it, so a +/// browser never restores it. +async fn list( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_plain_user_token(&authed)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(ListResponse { + enabled: false, + storage_id: None, + backup_generation: None, + fallback: false, + sessions: vec![], + truncated: false, + })); + }; + let prefix = backend.index_prefix(); + let cutoff = backend.retention_cutoff(); + let mut stream = backend.store.list(Some(&prefix)); + // One marker per session, whatever the session holds: the newest LIST_MAX are kept as + // the scan goes (a min-heap drops the oldest), and the scan itself is bounded. + let mut newest: std::collections::BinaryHeap< + std::cmp::Reverse<(chrono::DateTime, u32, String)>, + > = Default::default(); + let mut scanned = 0; + let mut truncated = false; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + scanned += 1; + if scanned > MAX_LIST_SCAN { + truncated = true; + break; + } + if cutoff.is_some_and(|cutoff| meta.last_modified < cutoff) { + continue; + } + let Some((sid, IndexEntry::Marker(epoch))) = index_entry(&prefix, &meta.location) else { + continue; + }; + newest.push(std::cmp::Reverse(( + meta.last_modified, + epoch, + sid.to_string(), + ))); + if newest.len() > LIST_MAX { + newest.pop(); + truncated = true; + } + } + let mut sessions: Vec = newest + .into_iter() + .map(|std::cmp::Reverse((updated_at, epoch, id))| SessionListing { id, updated_at, epoch }) + .collect(); + sessions.sort_by(|a, b| b.updated_at.cmp(&a.updated_at)); + Ok(Json(ListResponse { + enabled: true, + storage_id: Some(backend.storage_id.clone()), + backup_generation: Some(backend.generation), + fallback: backend.fallback, + sessions, + truncated, + })) +} + +#[derive(Deserialize)] +struct PullRequest { + ids: Vec, + /// Picks the session an earlier answer cut up from where it stopped; `ids` then names + /// that session alone. + #[serde(default)] + resume: Option, +} + +/// Where a pull of a session that outgrew one answer picks up: the last key the earlier +/// answer carried, in the session's prefix or, once that one is done, in its images prefix. +#[derive(Serialize, Deserialize, Clone)] +struct PullCursor { + id: String, + images: bool, + after: String, +} + +#[derive(Serialize)] +struct PulledChat { + id: String, + record: Box, +} + +#[derive(Serialize, Deserialize)] +struct ImageObject { + chat_id: String, + id: String, + data_url: String, +} + +#[derive(Serialize)] +struct PulledSession { + id: String, + head: Box, + chats: Vec, + images: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + artifacts: Option>, + /// The session did not fit this answer whole: the rest follows a pull with this cursor. + #[serde(skip_serializing_if = "Option::is_none")] + next: Option, + /// A fingerprint of the session's listing (marker, and every key, size, modification + /// time, entity tag and version), so the browser tells that the backup changed between + /// the pages it assembled. + listing: String, + /// The backup kept changing while this page was read (a push landing object by object), + /// so the page may mix two versions: the browser starts the session over. + #[serde(skip_serializing_if = "std::ops::Not::not")] + moved: bool, +} + +/// How many times a page whose listing moved while it was read is read again before it is +/// handed over as `moved`. +const PULL_REREADS: usize = 3; + +#[derive(Serialize)] +struct PullResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, + sessions: Vec, + /// Ids that did not fit the response budget; ask for them again. + deferred: Vec, +} + +fn raw(kind: &str, text: String) -> Result> { + RawValue::from_string(text) + .map_err(|e| Error::internal_err(format!("stored {kind} is not JSON: {e}"))) +} + +enum PullStep { + Absent, + Deferred, + Fetched(PulledSession, usize), +} + +/// Fetch one session: its head, then every chat and artifact object under its prefix, and +/// as many of its images as the budget allows (a missing image hydrates to a placeholder +/// in the browser). Sizes come from the listings, so a session that would not fit is +/// deferred before anything of it is read, unless it is the first of the response, which +/// must carry something. `Absent` when it has no head. +async fn pull_session( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // A page read while a push lands object by object may mix two versions of the session: + // the listing is taken again once the page is read, and a page it moved under is read + // again, a few times, then handed over as such for the browser to start over. + for reread in 0..PULL_REREADS { + let step = pull_page(backend, sid, budget, first, resume).await?; + let PullStep::Fetched(mut page, size) = step else { + return Ok(step); + }; + if backend.listing_fingerprint(sid).await?.as_deref() == Some(page.listing.as_str()) { + return Ok(PullStep::Fetched(page, size)); + } + if reread + 1 == PULL_REREADS { + page.moved = true; + return Ok(PullStep::Fetched(page, size)); + } + } + unreachable!("a page is answered on the last reread") +} + +async fn pull_page( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // Taken before anything of the page is listed or read: an object landing after it is + // in the next page's fingerprint, whereas one landing after the reads but before a + // fingerprint taken then would have certified a page without it. A session the storage + // does not list (removed, or a whole push in progress) is absent. + let Some(listing) = backend.listing_fingerprint(sid).await? else { + return Ok(PullStep::Absent); + }; + let Read::Text(head) = backend + .get(&backend.head_key(sid), MAX_HEAD_BYTES + CIPHER_PADDING) + .await? + else { + return Ok(PullStep::Absent); + }; + let session_prefix = backend.session_prefix(sid); + let images_prefix = backend.images_prefix(sid); + let mut size = head.len(); + let mut chats = vec![]; + let mut artifacts = None; + let mut images = vec![]; + let mut next = None; + let cursor = |images: bool, after: String| PullCursor { id: sid.to_string(), images, after }; + // Sizes come from the listings, and the listings stop at the budget, so nothing is read + // past it even for the first session of the answer. One that outgrew it (chats + // accumulate over pushes) comes back in pages, in key order, each answer naming where + // the next picks up; the browser imports nothing before the last page. An object that + // grew since the listing (a push replaced it) ends the page just before it, and the + // answer names that spot: a new listing sizes it, whereas dropping it would import the + // session without it for good. + let in_images = resume.is_some_and(|c| c.images); + if !in_images { + let after = resume.map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &session_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + if cut && !first { + return Ok(PullStep::Deferred); + } + if cut { + next = entries + .last() + .map(|(key, _)| cursor(false, key.to_string())); + } + let to_read: Vec<(ObjectPath, usize, Option)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key + .as_ref() + .strip_prefix(session_prefix.as_ref()) + .unwrap_or_default() + .trim_start_matches('/'); + if rel == "artifacts.json" { + Some((key, bytes, None)) + } else { + let cid = rel + .strip_prefix("chats/")? + .strip_suffix(".json")? + .to_string(); + Some((key, bytes, Some(cid))) + } + }) + .collect(); + let reads: Vec<(ObjectPath, usize, Option, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + let mut before = resume.map(|c| c.after.clone()).unwrap_or_default(); + for (key, bytes, cid, read) in reads { + match (cid, read) { + (_, Read::Grown) => { + next = Some(cursor(false, before)); + break; + } + (None, Read::Text(text)) => { + artifacts = Some(raw("artifacts", text)?); + size += bytes; + } + (Some(cid), Read::Text(text)) => { + chats.push(PulledChat { id: cid, record: raw("chat", text)? }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + if next.is_none() { + let after = resume + .filter(|c| c.images) + .map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &images_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + let mut before = after.map(|a| a.to_string()).unwrap_or_default(); + if cut { + // An answer with no room for a single image names where it stood, so the pull + // owed the session alone picks it up there. + next = Some(cursor( + true, + entries + .last() + .map(|(key, _)| key.to_string()) + .unwrap_or_else(|| before.clone()), + )); + } + let to_read: Vec<(ObjectPath, usize, String, String)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key.as_ref().strip_prefix(images_prefix.as_ref())?; + let (cid, iid) = rel.trim_start_matches('/').split_once('/')?; + Some((key.clone(), bytes, cid.to_string(), iid.to_string())) + }) + .collect(); + let reads: Vec<(ObjectPath, usize, String, String, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid, iid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, iid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + for (key, bytes, chat_id, id, read) in reads { + match read { + Read::Grown => { + next = Some(cursor(true, before)); + break; + } + Read::Text(data_url) => { + images.push(ImageObject { chat_id, id, data_url }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + Ok(PullStep::Fetched( + PulledSession { + id: sid.to_string(), + head: raw("head", head)?, + chats, + images, + artifacts, + next, + listing, + moved: false, + }, + size, + )) +} + +async fn pull( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.ids.len() > MAX_PULL_IDS { + return Err(Error::BadRequest(format!( + "at most {MAX_PULL_IDS} sessions per pull" + ))); + } + for id in &req.ids { + require_valid_id("session", id)?; + } + if let Some(cursor) = &req.resume { + if req.ids.len() != 1 || req.ids[0] != cursor.id || cursor.after.len() > 1024 { + return Err(Error::BadRequest( + "a resumed pull names the resumed session alone".to_string(), + )); + } + } + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PullResponse { + enabled: false, + storage_id: None, + backup_generation: None, + fallback: false, + sessions: vec![], + deferred: vec![], + })); + }; + let mut sessions = vec![]; + let mut deferred = vec![]; + let mut budget = PULL_RESPONSE_BUDGET; + for sid in req.ids { + let resume = req.resume.as_ref().filter(|c| c.id == sid); + match pull_session(&backend, &sid, budget, sessions.is_empty(), resume).await? { + PullStep::Absent => {} + PullStep::Deferred => deferred.push(sid), + PullStep::Fetched(session, size) => { + budget = budget.saturating_sub(size); + sessions.push(session); + } + } + } + Ok(Json(PullResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + fallback: backend.fallback, + sessions, + deferred, + })) +} + +#[derive(Deserialize)] +struct PushedChat { + id: String, + record: Box, +} + +#[derive(Deserialize)] +struct ImageRef { + chat_id: String, + id: String, +} + +#[derive(Deserialize)] +struct PushedSession { + id: String, + #[serde(default)] + head: Option>, + #[serde(default)] + chats: Vec, + #[serde(default)] + images: Vec, + #[serde(default)] + artifacts: Option>, + #[serde(default)] + delete_chats: Vec, + #[serde(default)] + delete_images: Vec, + /// More parts of the session follow, in this push or a later one: the session is not + /// listed on this one. + #[serde(default)] + partial: bool, + /// A part of a push of the session whole: the head is on the part that opens it, which + /// replaces whatever the storage holds of the session, and every piece the browser has + /// is on one of them. An incremental part instead rides on a session the storage lists, + /// and is refused with `needs_whole` when it lists none. + #[serde(default)] + whole: bool, + /// A push split over several parts names itself on each of them with a token the + /// browser draws; the part that `opens` it unlists the session (a pull between two parts + /// would otherwise take a mix of old and new pieces for the backup) and the last part + /// lists it again. A later part is written only while that token is the one there, so a + /// part of a push another one superseded is refused with `needs_whole`. + #[serde(default)] + push: Option, + #[serde(default)] + opens: bool, + /// The session's move count (its record's `moves`), the marker that lists the session + /// is named by: a session moved to another workspace is listed by both until the old + /// copy's removal lands, and the copy with the higher count is the later one. An + /// incremental part rides on the marker of the same count. + #[serde(default)] + epoch: u32, +} + +#[derive(Deserialize)] +struct PushRequest { + /// The email the browser believes it is acting for. An in-place account switch can + /// leave a flush prepared for the previous user; the server refuses it rather than + /// filing that user's sessions under the caller's prefix. + owner: String, + #[serde(default)] + sessions: Vec, + #[serde(default)] + removed: Vec, +} + +#[derive(Serialize)] +struct PushResult { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + /// Nothing was written; the session must be pushed whole again: an incremental part + /// found no listed session to ride on (another device removed the backup, or a push + /// split over parts is in progress or was abandoned), or a later part of a push split + /// over parts found another push had superseded it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + needs_whole: bool, +} + +#[derive(Serialize)] +struct PushResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, + results: Vec, +} + +fn validate_push(req: &PushRequest) -> Result<()> { + if req.sessions.len() > MAX_PUSH_SESSIONS || req.removed.len() > MAX_REMOVED { + return Err(Error::BadRequest( + "too many sessions in one push".to_string(), + )); + } + // Every nested entry costs an object-store call (a deleted chat two), so the lists are + // bounded per entry and across the request; the browser sends far fewer. + let mut operations = req.removed.len(); + for s in &req.sessions { + if s.chats.len() > MAX_CHATS_PER_ENTRY + || s.images.len() > MAX_IMAGES_PER_ENTRY + || s.delete_chats.len() > MAX_DELETES_PER_ENTRY + || s.delete_images.len() > MAX_DELETES_PER_ENTRY + { + return Err(Error::BadRequest(format!( + "too many pieces for session {} in one push", + s.id + ))); + } + operations += s.chats.len() + s.images.len() + s.delete_chats.len() + s.delete_images.len(); + } + if operations > MAX_OPERATIONS_PER_PUSH { + return Err(Error::BadRequest("too many pieces in one push".to_string())); + } + for sid in &req.removed { + require_valid_id("session", sid)?; + } + for s in &req.sessions { + require_valid_id("session", &s.id)?; + if let Some(token) = &s.push { + require_valid_id("push", token)?; + } else if s.opens || s.partial { + // A part more parts follow belongs to a push split over parts, which names + // itself: without the token the session would stay listed between the parts. + return Err(Error::BadRequest(format!( + "session {} is pushed in parts with no push token", + s.id + ))); + } + if s.whole && (s.push.is_none() || s.opens) && s.head.is_none() { + return Err(Error::BadRequest(format!( + "session {} is pushed whole without its head", + s.id + ))); + } + if let Some(head) = &s.head { + require_json_object("head", head, MAX_HEAD_BYTES)?; + } + for c in &s.chats { + require_valid_id("chat", &c.id)?; + require_json_object("chat record", &c.record, PUSH_BODY_LIMIT)?; + } + if let Some(a) = &s.artifacts { + require_json_object("artifacts", a, PUSH_BODY_LIMIT)?; + } + for i in &s.images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + require_data_url(&i.data_url)?; + } + for c in &s.delete_chats { + require_valid_id("chat", c)?; + } + for i in &s.delete_images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + } + } + Ok(()) +} + +fn push_payload_bytes(req: &PushRequest) -> usize { + req.sessions + .iter() + .map(|s| { + s.head.as_ref().map_or(0, |h| h.get().len()) + + s.artifacts.as_ref().map_or(0, |a| a.get().len()) + + s.chats.iter().map(|c| c.record.get().len()).sum::() + + s.images.iter().map(|i| i.data_url.len()).sum::() + }) + .sum() +} + +/// Runs under the session's lock (see `lock_session`). The part that opens a whole push +/// (the one with the head) replaces the backup: the marker goes first, so nothing lists the +/// session until the last part, then everything else. An incremental part assumes the rest +/// of the session is in the storage, which a removal since would have taken, or a push +/// split over parts may still be bringing: it is refused unless the session is listed, and +/// unlists the session itself while it changes more than one object (a pull between two +/// writes would otherwise take a mix of old and new pieces for the backup). A push split +/// over parts names itself with a token: the part that opens it unlists the session and +/// writes the token, and a later part is written only while that token is +/// the one there, so two devices pushing the session at once cannot list a mix of their +/// pieces: the push that opened later wins, the other is refused and goes again. Every +/// refusal comes before anything of the part lands. Deletes run last, and the marker only +/// by the last part, so a push cut short never leaves a listed session pointing at chats +/// that are not there. +/// Bytes written, and whether the part was refused for the session to go whole. +async fn push_session(backend: &Backend, s: &PushedSession) -> Result<(usize, bool)> { + match &s.push { + Some(token) if !s.opens => { + match backend + .get(&backend.push_key(&s.id), token.len() + CIPHER_PADDING) + .await? + { + Read::Text(current) if current == *token => {} + _ => return Ok((0, true)), + } + } + _ => { + if s.whole { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + backend + .delete_prefix(&backend.session_prefix(&s.id)) + .await?; + backend.delete_prefix(&backend.images_prefix(&s.id)).await?; + } else { + if !backend.exists(&backend.index_key(&s.id, s.epoch)).await? { + return Ok((0, true)); + } + // Unlisted while more than one object changes (a push split over parts, or + // one part touching several pieces): a pull between two of the writes, or + // after one of them failed, would otherwise take a mix of old and new + // pieces for the backup. One object changing is one write. + let pieces = s.chats.len() + + s.images.len() + + usize::from(s.artifacts.is_some()) + + usize::from(s.head.is_some()) + + s.delete_chats.len() + + s.delete_images.len(); + if s.push.is_some() || pieces > 1 { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + } + } + if let Some(token) = &s.push { + backend + .put(&backend.push_key(&s.id), token.as_bytes()) + .await?; + } + } + } + let mut written = 0; + written += backend + .put_all( + s.images + .iter() + .map(|img| { + ( + backend.image_key(&s.id, &img.chat_id, &img.id), + backend.seal(img.data_url.as_bytes()), + ) + }) + .collect(), + ) + .await?; + written += backend + .put_all( + s.chats + .iter() + .map(|c| { + ( + backend.chat_key(&s.id, &c.id), + backend.seal(c.record.get().as_bytes()), + ) + }) + .collect(), + ) + .await?; + if let Some(a) = &s.artifacts { + written += backend + .put(&backend.artifacts_key(&s.id), a.get().as_bytes()) + .await?; + } + if let Some(h) = &s.head { + written += backend + .put(&backend.head_key(&s.id), h.get().as_bytes()) + .await?; + } + for cid in &s.delete_chats { + backend.delete(&backend.chat_key(&s.id, cid)).await?; + backend + .delete_prefix(&backend.chat_images_prefix(&s.id, cid)) + .await?; + } + backend + .delete_all( + s.delete_images + .iter() + .map(|i| backend.image_key(&s.id, &i.chat_id, &i.id)) + .collect(), + ) + .await?; + if s.partial { + return Ok((written, false)); + } + // Last, and by the last part only, so a session is listed once its whole entry landed. + backend + .store + .put(&backend.index_key(&s.id, s.epoch), PutPayload::new()) + .await + .map_err(object_store_error_to_error)?; + if s.push.is_some() { + backend.delete(&backend.push_key(&s.id)).await?; + } + Ok((written, false)) +} + +/// The markers go first so a removal cut short leaves nothing listed, then the head so +/// nothing pulls either, and no push takes it for a session still there (see `push_session`). +/// The retention sweep's record goes last (see `Backend::sweep_key`). +async fn remove_session(backend: &Backend, sid: &str) -> Result<()> { + let sweep = backend.sweep_key(sid); + backend + .store + .list(Some(&backend.index_session_prefix(sid))) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| { + let sweep = &sweep; + async move { + if meta.location == *sweep { + return Ok(()); + } + backend.delete(&meta.location).await + } + }) + .await?; + backend.delete(&backend.head_key(sid)).await?; + backend.delete_prefix(&backend.session_prefix(sid)).await?; + backend.delete_prefix(&backend.images_prefix(sid)).await?; + backend.delete(&sweep).await +} + +/// One writer per session at a time, across servers: a push and a removal of the same +/// session interleaving object by object could leave a listed session missing pieces, or a +/// marker over nothing. The lock lives in a transaction that writes no rows; it is released +/// when the transaction ends. A wait past the timeout fails that entry only, and the +/// browser retries it with backoff. +async fn lock_session( + db: &DB, + backend: &Backend, + sid: &str, +) -> Result> { + let mut tx = db.begin().await?; + sqlx::query("SET LOCAL lock_timeout = '30s'") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0::int8))") + .bind(format!("ai_session_backup:{}/{sid}", backend.prefix)) + .execute(&mut *tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "another device is writing the backup of session {sid}; retried later ({e})" + )) + })?; + Ok(tx) +} + +async fn push_session_locked( + db: &DB, + backend: &Backend, + s: &PushedSession, +) -> Result<(usize, bool)> { + let tx = lock_session(db, backend, &s.id).await?; + let result = push_session(backend, s).await; + tx.commit().await?; + result +} + +async fn remove_session_locked(db: &DB, backend: &Backend, sid: &str) -> Result<()> { + let tx = lock_session(db, backend, sid).await?; + let result = remove_session(backend, sid).await; + tx.commit().await?; + result +} + +/// Deletes, in every workspace with `ai_config.sessions_retention_days`, the backups of the +/// sessions whose marker is older than that: the marker is rewritten by every push that +/// completes, so its modification time is the session's last activity as the storage clocks +/// it. For the monitor, on every server: a session-level advisory lock keeps one pass at a +/// time across them. The walk reads markers only, one object per session and nothing of what +/// the sessions hold, under each user's prefix in turn (`list_with_delimiter` names the +/// users), and deletes at most `SWEEP_MAX_PER_WORKSPACE` sessions per workspace and pass. A +/// session goes under its lock (`lock_session`), once its markers are listed again there and +/// still all older (see `sweep_session`). A removal cut short leaves the sweep's record next +/// to the markers, which the walk also collects, so the next pass finishes it. +pub async fn sweep_expired_ai_session_backups(db: &DB) { + let mut lock_conn = match db.acquire().await { + Ok(conn) => conn, + Err(e) => { + tracing::error!("AI session retention: could not acquire a connection: {e:#}"); + return; + } + }; + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(SWEEP_LOCK_ID) + .fetch_one(&mut *lock_conn) + .await + { + Ok(locked) => locked, + Err(e) => { + tracing::error!("AI session retention: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + return; + } + if let Err(e) = sweep_workspaces(db).await { + tracing::error!("AI session retention sweep failed: {e:#}"); + } + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(SWEEP_LOCK_ID) + .execute(&mut *lock_conn) + .await + { + tracing::error!("AI session retention: advisory unlock failed: {e:#}"); + } +} + +async fn sweep_workspaces(db: &DB) -> Result<()> { + let workspaces = sqlx::query_as::<_, (String, Option, i64, bool)>( + "SELECT workspace_id, ai_config->'sessions_retention_days', ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL \ + FROM workspace_settings \ + WHERE ai_config->'sessions_retention_days' IS NOT NULL", + ) + .fetch_all(db) + .await?; + for (w_id, retention, generation, has_storage) in workspaces { + let Some(days) = sessions_retention_days(retention.as_ref()) else { + continue; + }; + match sweep_workspace(db, &w_id, days, generation, has_storage).await { + Ok(0) => {} + Ok(deleted) => tracing::info!( + "AI session retention deleted {deleted} session backups of {w_id} older than {days} days" + ), + Err(e) => tracing::warn!("AI session retention sweep of {w_id}: {e:#}"), + } + } + Ok(()) +} + +/// `has_storage` comes from the row `generation` was read from, as in `backend`: the instance +/// store is swept only under a generation read while the workspace had no storage of its own. +async fn sweep_workspace( + db: &DB, + w_id: &str, + days: u32, + generation: i64, + has_storage: bool, +) -> Result { + let resolved = if has_storage { + primary_store(db, w_id).await? + } else { + fallback_store(db).await? + }; + let Some(resolved) = resolved else { + return Ok(0); + }; + let key = get_workspace_key(w_id, db).await?; + let (store, storage_id) = (resolved.store, resolved.storage_id); + let cutoff = retention_cutoff(days); + let root = ObjectPath::from(generation_prefix(w_id, generation)); + let users = store + .list_with_delimiter(Some(&root)) + .await + .map_err(object_store_error_to_error)? + .common_prefixes; + let mut deleted = 0; + for user_prefix in users { + let Some(user) = user_prefix.filename() else { + continue; + }; + // The sweep decrypts nothing; the cipher is only what a `Backend` is made of. + let backend = Backend { + store: store.clone(), + mc: crypt_from_key_with_suffix(&key, user), + prefix: user_prefix.to_string(), + storage_id: storage_id.clone(), + generation, + retention_days: Some(days), + fallback: resolved.fallback, + }; + let index = backend.index_prefix(); + let mut markers = backend.store.list(Some(&index)); + let mut expired = std::collections::BTreeSet::new(); + while let Some(meta) = markers.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + let sid = match index_entry(&index, &meta.location) { + Some((sid, IndexEntry::Sweep)) => sid, + Some((sid, IndexEntry::Marker(_) | IndexEntry::Push)) + if meta.last_modified < cutoff => + { + sid + } + _ => continue, + }; + expired.insert(sid.to_string()); + if deleted + expired.len() >= SWEEP_MAX_PER_WORKSPACE { + break; + } + } + for sid in expired { + match sweep_session(db, &backend, &sid, cutoff).await { + Ok(true) => deleted += 1, + Ok(false) => {} + Err(e) => tracing::warn!( + "AI session retention left the backup of {sid} in {w_id} for the next pass: {e:#}" + ), + } + } + if deleted >= SWEEP_MAX_PER_WORKSPACE { + break; + } + } + Ok(deleted) +} + +/// True when the session was deleted. Under the session's lock its markers are listed again: +/// one a push renewed since the walk keeps the session. A session with none is left alone +/// while a push split over parts is between two of them (its token younger than the +/// retention) or it is gone, unless the sweep's record says a removal was started; an older +/// token is a split push a browser abandoned, whose landed parts nothing lists. The record is +/// written before anything is deleted and removed last, so a removal cut short is found again +/// by the next pass. +async fn sweep_session( + db: &DB, + backend: &Backend, + sid: &str, + cutoff: chrono::DateTime, +) -> Result { + let tx = lock_session(db, backend, sid).await?; + let result = async { + let (sweep, push) = (backend.sweep_key(sid), backend.push_key(sid)); + let mut entries = backend.store.list(Some(&backend.index_session_prefix(sid))); + let (mut listed, mut renewed, mut started, mut abandoned) = (false, false, false, false); + while let Some(meta) = entries.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + if meta.location == sweep { + started = true; + } else if meta.location == push { + abandoned = meta.last_modified < cutoff; + } else { + listed = true; + renewed |= meta.last_modified >= cutoff; + } + } + if renewed { + // A push listed the session again over a removal cut short before its markers + // went, which had deleted nothing else. + if started { + backend.delete(&sweep).await?; + } + return Ok(false); + } + if !listed && !started && !abandoned { + return Ok(false); + } + if !started { + backend + .store + .put(&sweep, PutPayload::new()) + .await + .map_err(object_store_error_to_error)?; + } + remove_session(backend, sid).await?; + Ok(true) + } + .await; + tx.commit().await?; + result +} + +async fn push( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.owner != authed.email { + return Err(Error::Generic( + http::StatusCode::CONFLICT, + "this push was prepared for another user".to_string(), + )); + } + validate_push(&req)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PushResponse { + enabled: false, + storage_id: None, + backup_generation: None, + fallback: false, + results: vec![], + })); + }; + #[cfg(not(feature = "enterprise"))] + { + let remaining = + crate::job_helpers_oss::ce_storage_quota_remaining(&db, &w_id, None).await?; + if push_payload_bytes(&req) as i64 > remaining { + return Err(Error::QuotaExceeded( + "the workspace storage quota leaves no room for this AI session backup".to_string(), + )); + } + } + #[cfg(feature = "enterprise")] + let _ = push_payload_bytes(&req); + + let mut results = Vec::with_capacity(req.sessions.len() + req.removed.len()); + let mut written: usize = 0; + // A session split into several entries is listed by the last: once one part failed, the + // later ones are not written, or the marker would list a session missing a part. + let mut failed: std::collections::HashSet<&str> = Default::default(); + for s in &req.sessions { + let (error, needs_whole) = if failed.contains(s.id.as_str()) { + ( + Some("an earlier part of this session in the push failed".to_string()), + false, + ) + } else { + match push_session_locked(&db, &backend, s).await { + Ok((n, needs_whole)) => { + written += n; + (None, needs_whole) + } + Err(e) => { + tracing::warn!("AI session backup push failed for {} in {w_id}: {e}", s.id); + failed.insert(&s.id); + (Some(e.to_string()), false) + } + } + }; + results.push(PushResult { id: s.id.clone(), error, needs_whole }); + } + for sid in &req.removed { + let error = remove_session_locked(&db, &backend, sid) + .await + .err() + .map(|e| { + tracing::warn!("AI session backup removal failed for {sid} in {w_id}: {e}"); + e.to_string() + }); + results.push(PushResult { id: sid.clone(), error, needs_whole: false }); + } + // Overwrites and deletes make this an over-count; the periodic recount the quota check + // schedules once usage is stale settles it. Bytes in the instance store count under a + // name of their own, which the recount lists there. + #[cfg(not(feature = "enterprise"))] + if written > 0 { + let storage = if backend.fallback { + windmill_api_workspaces::ai_session_backups::FALLBACK_STORAGE + } else { + windmill_object_store::DEFAULT_STORAGE + }; + crate::job_helpers_oss::bump_storage_usage(&db, &w_id, storage, written as i64).await; + } + #[cfg(feature = "enterprise")] + let _ = written; + Ok(Json(PushResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + fallback: backend.fallback, + results, + })) +} diff --git a/backend/windmill-api/src/ai_shared_artifacts.rs b/backend/windmill-api/src/ai_shared_artifacts.rs new file mode 100644 index 0000000000..36ad0ecd2e --- /dev/null +++ b/backend/windmill-api/src/ai_shared_artifacts.rs @@ -0,0 +1,322 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Read-only copies of AI session artifacts, shared with the workspace by their author. +//! +//! Artifacts live in the author's browser; a row here exists only because the author asked +//! for a link. Every read filters on the retention window as well as the monitor sweeping +//! it, so a share past its window is never served in the gap before the sweep reaches it. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Extension, Json, Path, Query}, + routing::{delete, get, post}, + Router, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + ai_shared_artifact_retention_secs, + error::{Error, JsonResult, Result}, +}; + +/// The frontend's `MAX_ARTIFACT_BYTES`: no artifact the viewer holds is larger. +const MAX_CONTENT_BYTES: usize = 256 * 1024; +const MAX_NAME_CHARS: usize = 255; +const MAX_ARTIFACT_ID_CHARS: usize = 255; +/// JSON escapes a control character into six bytes, so a full-size artifact made entirely of +/// them still fits. +const SHARE_BODY_LIMIT: usize = MAX_CONTENT_BYTES * 6 + 64 * 1024; + +pub fn workspaced_service() -> Router { + Router::new() + .route( + "/share", + post(share_artifact).layer(DefaultBodyLimit::max(SHARE_BODY_LIMIT)), + ) + .route("/status", get(get_share_status)) + .route("/get/{id}", get(get_shared_artifact)) + .route("/delete/{id}", delete(unshare_artifact)) +} + +#[derive(Deserialize, Clone, Copy)] +#[serde(rename_all = "lowercase")] +enum ArtifactKind { + Md, + Html, +} + +impl ArtifactKind { + fn as_str(self) -> &'static str { + match self { + ArtifactKind::Md => "md", + ArtifactKind::Html => "html", + } + } +} + +#[derive(Deserialize)] +struct ShareArtifact { + artifact_id: String, + name: String, + kind: ArtifactKind, + version: i32, + content: String, +} + +#[derive(Serialize)] +struct SharedArtifactInfo { + id: Uuid, + name: String, + kind: String, + version: i32, + created_by: String, + shared_at: DateTime, + expires_at: DateTime, +} + +#[derive(Serialize)] +struct SharedArtifact { + #[serde(flatten)] + info: SharedArtifactInfo, + content: String, + /// Whether the caller may stop sharing it: its author, or a workspace admin. + can_unshare: bool, +} + +#[derive(Serialize)] +struct ShareStatus { + retention_secs: i64, + #[serde(skip_serializing_if = "Option::is_none")] + share: Option, +} + +#[derive(Deserialize)] +struct ShareStatusQuery { + artifact_id: String, +} + +fn expires_at(shared_at: DateTime, retention_secs: i64) -> DateTime { + shared_at + chrono::Duration::seconds(retention_secs) +} + +/// The browser-side artifact id, as every handler that takes one must check it: it is compared +/// against a `VARCHAR(255)` column, and Postgres answers a NUL in a text parameter with an +/// opaque 500. +fn check_artifact_id(artifact_id: &str) -> Result<()> { + if artifact_id.is_empty() || artifact_id.chars().count() > MAX_ARTIFACT_ID_CHARS { + return Err(Error::BadRequest(format!( + "Artifact id must be between 1 and {MAX_ARTIFACT_ID_CHARS} characters" + ))); + } + if artifact_id.contains('\0') { + return Err(Error::BadRequest( + "Artifact id cannot contain NUL characters".to_string(), + )); + } + Ok(()) +} + +/// Share an artifact, or move the caller's existing link for it to this content. Re-sharing +/// keeps the link and restarts its retention window. +async fn share_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> JsonResult { + let name = payload.name.trim(); + if name.is_empty() || name.chars().count() > MAX_NAME_CHARS { + return Err(Error::BadRequest(format!( + "Artifact name must be between 1 and {MAX_NAME_CHARS} characters" + ))); + } + check_artifact_id(&payload.artifact_id)?; + if payload.content.len() > MAX_CONTENT_BYTES { + return Err(Error::BadRequest(format!( + "Artifact content is {} bytes, above the {MAX_CONTENT_BYTES} byte limit", + payload.content.len() + ))); + } + if payload.version < 1 { + return Err(Error::BadRequest( + "Artifact version must be at least 1".to_string(), + )); + } + // Postgres rejects NUL in text columns with an opaque 500. + if name.contains('\0') || payload.content.contains('\0') { + return Err(Error::BadRequest( + "Artifact name and content cannot contain NUL characters".to_string(), + )); + } + + let mut tx = db.begin().await?; + let row = sqlx::query!( + r#"INSERT INTO ai_shared_artifact + (workspace_id, artifact_id, email, created_by, name, kind, version, content) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE + SET created_by = EXCLUDED.created_by, + name = EXCLUDED.name, + kind = EXCLUDED.kind, + version = EXCLUDED.version, + content = EXCLUDED.content, + shared_at = now() + RETURNING id, shared_at, (xmax = 0) AS "inserted!""#, + &w_id, + &payload.artifact_id, + &authed.email, + &authed.username, + name, + payload.kind.as_str(), + payload.version, + &payload.content, + ) + .fetch_one(&mut *tx) + .await?; + + let id = row.id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.share", + if row.inserted { + ActionKind::Create + } else { + ActionKind::Update + }, + &w_id, + Some(&id), + Some([("name", name)].into()), + ) + .await?; + tx.commit().await?; + + Ok(Json(SharedArtifactInfo { + id: row.id, + name: name.to_string(), + kind: payload.kind.as_str().to_string(), + version: payload.version, + created_by: authed.username.clone(), + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, ai_shared_artifact_retention_secs()), + })) +} + +/// The caller's own live share of one of their artifacts, if any, and how long a share lasts. +async fn get_share_status( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + check_artifact_id(&query.artifact_id)?; + let retention_secs = ai_shared_artifact_retention_secs(); + let share = sqlx::query!( + "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact + WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3 + AND shared_at > now() - ($4::bigint::text || ' s')::interval", + &w_id, + &authed.email, + &query.artifact_id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .map(|r| SharedArtifactInfo { + id: r.id, + name: r.name, + kind: r.kind, + version: r.version, + created_by: r.created_by, + shared_at: r.shared_at, + expires_at: expires_at(r.shared_at, retention_secs), + }); + + Ok(Json(ShareStatus { retention_secs, share })) +} + +/// Any member of the workspace may read a live share: that is what sharing it granted. +async fn get_shared_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> JsonResult { + let retention_secs = ai_shared_artifact_retention_secs(); + let row = sqlx::query!( + "SELECT id, email, name, kind, version, created_by, content, shared_at + FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 + AND shared_at > now() - ($3::bigint::text || ' s')::interval", + &w_id, + id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found: it may have expired or been unshared" + )) + })?; + + Ok(Json(SharedArtifact { + can_unshare: row.email == authed.email || authed.is_admin, + content: row.content, + info: SharedArtifactInfo { + id: row.id, + name: row.name, + kind: row.kind, + version: row.version, + created_by: row.created_by, + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, retention_secs), + }, + })) +} + +async fn unshare_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> Result { + let mut tx = db.begin().await?; + let name = sqlx::query_scalar!( + "DELETE FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool) + RETURNING name", + &w_id, + id, + &authed.email, + authed.is_admin, + ) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found, or not shared by you" + )) + })?; + + let id_str = id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.unshare", + ActionKind::Delete, + &w_id, + Some(&id_str), + Some([("name", name.as_str())].into()), + ) + .await?; + tx.commit().await?; + + Ok(format!("Stopped sharing {name}")) +} diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e5b5f019eb..717be4ad0f 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -17,6 +17,7 @@ use std::str::FromStr; use uuid::Uuid; use windmill_common::cache; use windmill_common::error::Error; +use windmill_common::flows::{ApprovalSkin, Suspend}; use windmill_common::jobs::JobKind; use windmill_common::scripts::ScriptHash; @@ -94,6 +95,17 @@ pub struct ApprovalFormDetails { pub message_str: String, pub urls: ResumeUrls, pub schema: Option, + pub skin: ApprovalSkin, +} + +/// The suspended step an approval message is about, and the flow run it belongs to. +struct ApprovalStep { + created_by: String, + created_at: chrono::NaiveDateTime, + script_path: Option, + parent_job_id: Option, + args: Option>>, + suspend: Option, } #[allow(dead_code)] @@ -205,6 +217,90 @@ pub async fn get_approval_form_details( tracing::debug!("Job ID: {:?}", job_id); + let ApprovalStep { created_by, created_at, script_path, parent_job_id, args, suspend } = + fetch_approval_step(&db, w_id, job_id, flow_step_id).await?; + + let schema = suspend.as_ref().map(|suspend| ResumeFormRow { + resume_form: suspend.resume_form.clone(), + hide_cancel: suspend.hide_cancel, + }); + let skin = suspend.and_then(|s| s.skin).unwrap_or_default(); + + let bold_format = match format { + MessageFormat::Slack => "*{}*", + MessageFormat::Teams => "**{}**", + }; + + let message_str = match skin { + ApprovalSkin::Detailed => { + let args_str = args.map_or("None".to_string(), |a| { + serde_json::from_str::(a.get()) + .ok() + .and_then(|v| serde_json::to_string_pretty(&v).ok()) + .unwrap_or_else(|| a.get().to_string()) + }); + let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); + let script_path_str = script_path.as_deref().unwrap_or("None"); + + let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); + + let mut message_str = format!( + "A workflow has been suspended and is waiting for approval:\n\n\ + {}: {created_by}\n\n\ + {}: {created_at_formatted}\n\n\ + {}: {script_path_str}\n\n\ + {}:\n```\n{args_str}\n```\n\n\ + {}: {parent_job_id_str}\n\n", + bold_format.replace("{}", "Created by"), + bold_format.replace("{}", "Created at"), + bold_format.replace("{}", "Script path"), + bold_format.replace("{}", "Args"), + bold_format.replace("{}", "Flow ID") + ); + + // Append custom message if provided + if let Some(msg) = message { + message_str.push_str(msg); + } + message_str + } + ApprovalSkin::Minimal => format!( + "{}\n\n{}: {created_by}", + message.unwrap_or("Your approval is requested."), + bold_format.replace("{}", "Requested by"), + ), + }; + + tracing::debug!("Schema: {:#?}", schema); + + Ok(ApprovalFormDetails { message_str, urls, schema, skin }) +} + +/// The skin of the approval step `flow_step_id` of the flow running `job_id`. Falls back to +/// the detailed skin when the step cannot be resolved, so a message is still sent. +/// Reads through the unrestricted pool without an authorization check of its own: only the +/// skin, which is not sensitive, leaves this function. +pub(crate) async fn get_approval_step_skin( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: &str, +) -> ApprovalSkin { + match fetch_approval_step(db, w_id, job_id, Some(flow_step_id)).await { + Ok(step) => step.suspend.and_then(|s| s.skin).unwrap_or_default(), + Err(e) => { + tracing::warn!("Could not resolve approval step {flow_step_id} of job {job_id}: {e}"); + ApprovalSkin::default() + } + } +} + +async fn fetch_approval_step( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: Option<&str>, +) -> Result { // TODO: do we have a helper function for this? let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( "WITH job_info AS ( @@ -240,17 +336,17 @@ pub async fn get_approval_form_details( job_id, &w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await .map_err(|e| Error::BadRequest(e.to_string()))? .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string())) .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?; - let flow_data = match cache::job::fetch_flow(&db, &job_kind, script_hash).await { + let flow_data = match cache::job::fetch_flow(db, &job_kind, script_hash).await { Ok(data) => data, Err(_) => { if let Some(parent_job_id) = parent_job_id.as_ref() { - cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await? + cache::job::fetch_preview_flow(db, parent_job_id, raw_flow).await? } else { return Err(Error::BadRequest( "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(), @@ -265,49 +361,12 @@ pub async fn get_approval_form_details( tracing::debug!("Module: {:#?}", module); - let schema = module.and_then(|module| { - module.suspend.as_ref().map(|suspend| ResumeFormRow { - resume_form: suspend.resume_form.clone(), - hide_cancel: suspend.hide_cancel, - }) - }); - - let args_str = args.map_or("None".to_string(), |a| { - serde_json::from_str::(a.get()) - .ok() - .and_then(|v| serde_json::to_string_pretty(&v).ok()) - .unwrap_or_else(|| a.get().to_string()) - }); - let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); - let script_path_str = script_path.as_deref().unwrap_or("None"); - - let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); - - let bold_format = match format { - MessageFormat::Slack => "*{}*", - MessageFormat::Teams => "**{}**", - }; - - let mut message_str = format!( - "A workflow has been suspended and is waiting for approval:\n\n\ - {}: {created_by}\n\n\ - {}: {created_at_formatted}\n\n\ - {}: {script_path_str}\n\n\ - {}:\n```\n{args_str}\n```\n\n\ - {}: {parent_job_id_str}\n\n", - bold_format.replace("{}", "Created by"), - bold_format.replace("{}", "Created at"), - bold_format.replace("{}", "Script path"), - bold_format.replace("{}", "Args"), - bold_format.replace("{}", "Flow ID") - ); - - // Append custom message if provided - if let Some(msg) = message { - message_str.push_str(msg); - } - - tracing::debug!("Schema: {:#?}", schema); - - Ok(ApprovalFormDetails { message_str, urls, schema }) + Ok(ApprovalStep { + created_by, + created_at, + script_path, + parent_job_id, + args, + suspend: module.and_then(|m| m.suspend.clone()), + }) } diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index a674e6c38a..6af6bbc37c 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -343,6 +343,29 @@ fn refuse_unscopable_guest_app(path: &str, mode: ExecutionMode) -> Result<()> { Ok(()) } +/// Refuse *widening* an app into guests where the deployment has none +/// (`instance_supports_guests`). Only the transition is refused, like the protection +/// rule below it: an app already stored in the mode — deployed before the instance +/// became a cloud one, or pushed by git-sync — keeps deploying, and keeps being inert, +/// since every guest gate refuses it anyway. `deployed_mode` is what the app is stored +/// as, `None` when it is being created. +fn refuse_guest_mode_where_unavailable( + path: &str, + mode: ExecutionMode, + deployed_mode: Option, +) -> Result<()> { + if !matches!(mode, ExecutionMode::Guest) + || deployed_mode == Some(ExecutionMode::Guest) + || windmill_common::workspaces::instance_supports_guests() + { + return Ok(()); + } + Err(Error::BadRequest(format!( + "app {path} cannot be set to Guests: {}", + windmill_common::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))) +} + /// Gate a viewer on the app's `execution_mode`, as far as can be decided without an /// ACL probe. `Ok(true)` means already authorized — anonymous admits anyone, guest /// admits anyone signed in; `Ok(false)` means the caller is a member and still owes @@ -468,6 +491,13 @@ pub struct S3Key { #[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct Policy { pub on_behalf_of: Option, + /// The address `on_behalf_of` resolves to. Every write stores what the principal resolves + /// to, so it is not taken from the request except when a client names only the address — + /// which is how a cross-workspace deploy carries an identity — and it is rejected when the + /// two disagree. Optional: a policy without it executes by deriving from the principal, so + /// removing it is a change of default rather than of behavior — see + /// `docs/app-policy-email-removal.md`. + #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, //paths: // - script/ @@ -1410,12 +1440,14 @@ const APP_EMBED_TOKEN_VALIDITY_HOURS: i64 = 12; /// Scopes an app author may declare in `Policy::frontend_sdk_scopes`. No `apps:*` /// scope, so the token cannot reach the mint endpoints and renew itself; the /// `raw_app_sdk` sentinel narrows the rest (see `scopes.rs`). -pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 5] = [ +pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 7] = [ "jobs:run", "jobs:read", "users:read", "resources:read", "variables:read", + "flow_conversations:read", + "flow_conversations:write", ]; /// Reject a policy declaring frontend SDK scopes outside the curated list. @@ -2438,31 +2470,37 @@ async fn create_app_internal<'a>( } // Resolve the on-behalf-of defaults on the (non-RLS) pool *before* opening // the RLS transaction below: doing these lookups mid-transaction would hold - // a second simultaneous connection while `tx` is still checked out. + // a second simultaneous connection while `tx` is still checked out. The race this + // leaves with a concurrent rename or removal, including a freed username later + // rebinding the stored principal, is known and accepted: see `resolve_on_behalf_of`. let should_preserve = app.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed) - && app.policy.on_behalf_of.is_some(); + && (app.policy.on_behalf_of.is_some() || app.policy.on_behalf_of_email.is_some()); - if !should_preserve { + let mut preserved_on_behalf_of: Option = None; + if should_preserve { + app.policy.on_behalf_of = windmill_common::resolve_on_behalf_of( + app.policy.on_behalf_of_email.as_deref(), + app.policy.on_behalf_of.as_deref(), + true, + &authed, + w_id, + &db, + ) + .await?; + } else { let folder_default = if windmill_common::can_preserve_on_behalf_of(&authed) { windmill_common::folders::resolve_folder_default_permissioned_as(&db, w_id, &app.path) .await? } else { None }; - if let Some(default_permissioned_as) = folder_default { - let default_email = windmill_common::users::get_email_from_permissioned_as( - &default_permissioned_as, - w_id, - &db, - ) - .await?; - app.policy.on_behalf_of = Some(default_permissioned_as); - app.policy.on_behalf_of_email = Some(default_email); - } else { - app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - app.policy.on_behalf_of_email = Some(authed.email.clone()); - } + app.policy.on_behalf_of = + Some(folder_default.unwrap_or_else(|| username_to_permissioned_as(&authed.username))); + } + app.policy.on_behalf_of_email = stored_on_behalf_of_email(&app.policy, w_id, &db).await?; + if should_preserve { + preserved_on_behalf_of = audited_on_behalf_of(&app.policy, &authed); } // Reject a forged superadmin run identity in the (possibly preserved) policy. @@ -2518,6 +2556,7 @@ async fn create_app_internal<'a>( // even when the caller did not. app.policy.set_execution_mode(app.policy.execution_mode()); refuse_unscopable_guest_app(&app.path, app.policy.execution_mode())?; + refuse_guest_mode_where_unavailable(&app.path, app.policy.execution_mode(), None)?; if let Some(rule) = deployment_rule_for_mode(app.policy.execution_mode()) { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, @@ -2597,21 +2636,17 @@ async fn create_app_internal<'a>( None, ) .await?; - if should_preserve { - if let Some(ref obo_email) = app.policy.on_behalf_of_email { - if obo_email != &authed.email { - audit_log( - &mut *tx, - &authed, - "apps.on_behalf_of", - ActionKind::Create, - w_id, - Some(&app.path), - Some([("on_behalf_of", obo_email.as_str()), ("action", "create")].into()), - ) - .await?; - } - } + if let Some(ref obo_email) = preserved_on_behalf_of { + audit_log( + &mut *tx, + &authed, + "apps.on_behalf_of", + ActionKind::Create, + w_id, + Some(&app.path), + Some([("on_behalf_of", obo_email.as_str()), ("action", "create")].into()), + ) + .await?; } let mut args: HashMap> = HashMap::new(); if let Some(dm) = &app.deployment_message { @@ -3368,19 +3403,33 @@ async fn update_app_internal<'a>( } } - // Reject a forged superadmin run identity in a preserved policy. Mirror the - // `should_preserve` gate below (only a preserved value is caller-controlled; - // otherwise the policy is rewritten to the deployer's own identity) and run - // it on the non-RLS pool before the transaction to avoid a second connection. - if let Some(npolicy) = ns.policy.as_ref() { + // Resolved on the (non-RLS) pool before the RLS transaction opens, for the reason + // `create_app` states, with the same known, accepted rename race (see + // `resolve_on_behalf_of`). Submitting a policy is how a deployer claims the app's execution + // identity; a source deploy that sent none claims nothing, so whoever the app already runs as + // stays. + let mut preserved_on_behalf_of: Option = None; + if let Some(npolicy) = ns.policy.as_mut() { let should_preserve = ns.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed) - && npolicy.on_behalf_of.is_some(); + && (npolicy.on_behalf_of.is_some() || npolicy.on_behalf_of_email.is_some()); + if should_preserve { - windmill_common::auth::validate_on_behalf_of( - npolicy.on_behalf_of.as_deref(), + npolicy.on_behalf_of = windmill_common::resolve_on_behalf_of( npolicy.on_behalf_of_email.as_deref(), - )?; + npolicy.on_behalf_of.as_deref(), + true, + &authed, + w_id, + &db, + ) + .await?; + } else { + npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); + } + npolicy.on_behalf_of_email = stored_on_behalf_of_email(npolicy, w_id, &db).await?; + if should_preserve { + preserved_on_behalf_of = audited_on_behalf_of(npolicy, &authed); } } @@ -3413,7 +3462,6 @@ async fn update_app_internal<'a>( reject_kind_change(path, raw_app, deployed_raw_app)?; } - let mut preserved_on_behalf_of: Option = None; let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() @@ -3565,6 +3613,13 @@ async fn update_app_internal<'a>( ns.path.as_deref().unwrap_or(path), npolicy.execution_mode(), )?; + // An unreadable deployed policy reads as not already-in-mode, the strict + // direction, as for the protection rule below. + refuse_guest_mode_where_unavailable( + ns.path.as_deref().unwrap_or(path), + npolicy.execution_mode(), + deployed_policy.as_ref().map(|d| d.execution_mode()), + )?; if let Some(rule) = deployment_rule_for_mode(npolicy.execution_mode()).filter(|_| !authed.is_admin) { @@ -3592,23 +3647,6 @@ async fn update_app_internal<'a>( } } } - let should_preserve = ns.preserve_on_behalf_of.unwrap_or(false) - && windmill_common::can_preserve_on_behalf_of(&authed) - && npolicy.on_behalf_of.is_some(); - - if should_preserve { - if let Some(ref obo_email) = npolicy.on_behalf_of_email { - if obo_email != &authed.email { - preserved_on_behalf_of = Some(obo_email.clone()); - } - } - } else if caller_sent_policy { - // Submitting a policy is how a deployer claims the app's - // execution identity. A source deploy that sent none is not - // claiming anything, so whoever the app already runs as stays. - npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - npolicy.on_behalf_of_email = Some(authed.email.clone()); - } sqlb.set( "policy", quote(serde_json::to_string(&json!(npolicy)).map_err(|e| { @@ -3815,9 +3853,31 @@ fn digest(code: &str) -> String { format!("rawscript/{:x}", result) } +/// Canonical `runnable_path` for a run-mode no-id inline app component: +/// `/` — byte-for-byte what the runtime frontend sends. It is +/// the relative-import base, so the caller must not steer it: reject a `component` +/// that isn't a single non-empty path segment (empty, a separator, or `.`/`..` +/// would walk the base out of the app, and a bare `rawscript/` policy key +/// does not pin the component). +fn inline_run_path(app_path: &str, component: &str) -> Result { + if component.is_empty() + || component.contains('/') + || component.contains('\\') + || component == "." + || component == ".." + { + return Err(Error::BadRequest( + "component id must be a single non-empty path segment".to_string(), + )); + } + Ok(format!("{app_path}/{component}")) +} + async fn get_on_behalf_details_from_policy_and_authed( policy: &Policy, opt_authed: &Option, + w_id: &str, + db: &DB, ) -> Result<(String, String, String)> { // A guest acts only through an app open to guests — or to everyone. A members-only // mode means the policy changed after the session was issued. Decided here, in the @@ -3840,7 +3900,7 @@ async fn get_on_behalf_details_from_policy_and_authed( .as_ref() .map(|a| a.username.clone()) .unwrap_or_else(|| "anonymous".to_string()); - let (permissioned_as, email) = get_on_behalf_of(&policy)?; + let (permissioned_as, email) = get_on_behalf_of(&policy, w_id, db).await?; (username, permissioned_as, email) } // Guest runs as the publisher exactly as Publisher does; the two differ only @@ -3854,7 +3914,7 @@ async fn get_on_behalf_details_from_policy_and_authed( "publisher execution mode requires authentication".to_string(), ) })?; - let (permissioned_as, email) = get_on_behalf_of(&policy)?; + let (permissioned_as, email) = get_on_behalf_of(&policy, w_id, db).await?; (username, permissioned_as, email) } ExecutionMode::Viewer => { @@ -4212,11 +4272,19 @@ async fn execute_component( } let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); + // `_MODULES` and `_TEMP_SCRIPT_REFS` are server-injected control keys (into + // `extra`) that the worker reads back for a `Preview` job — which an inline run + // is. A caller supplying them in `args` would inject module content/locks or + // redirect relative-import resolution, unpinned, as the app identity. Drop them; + // legitimate values ride in `extra`, never the request `args`. + payload.args.remove("_MODULES"); + payload.args.remove("_TEMP_SCRIPT_REFS"); + let (mut args, job_id) = build_args( policy, policy_triggerables, @@ -4254,6 +4322,7 @@ async fn execute_component( } .filter(|t| !t.is_empty()) }; + let component = payload.component.clone(); let (job_payload, tag, _runnable_on_behalf_of) = match (payload.path, payload.raw_code, payload.id) { // flow or script: @@ -4264,6 +4333,29 @@ async fn execute_component( // `app_script` table (legacy `rawscript/`-keyed triggerables). (None, Some(raw_code), None) => { let tag = resolved_inline_tag(raw_code.tag.clone()); + let raw_code = if is_preview { + // Preview (editor / `wmill app dev`): the caller runs their own + // code, like `/jobs/run/preview` — honored verbatim. + raw_code + } else { + // Run mode. Legacy back-compat only: current deploys assign an + // `app_script` id (reduce_app) and take the `Some(id)` arm; + // drop this branch once id-less deployed apps are gone. + // + // Only `content` is pinned (`rawscript/`), so keep just + // that plus `language`/`cache_ttl`, derive `path` server-side + // (`inline_run_path`), and default the rest: a caller `hash`/ + // `lock`/`modules`/`path`/`dedicated_worker` would otherwise run + // or install unpinned code as the app identity. Reconstructing + // (vs nulling) keeps a new field defaulting safe. + RawCode { + content: raw_code.content, + language: raw_code.language, + path: Some(inline_run_path(path, &component)?), + cache_ttl: raw_code.cache_ttl, + ..Default::default() + } + }; (JobPayload::Code(raw_code), tag, None) } // inline script: run mode (deployed app) with an entry in `app_script`. @@ -4582,7 +4674,7 @@ async fn upload_s3_file_from_app( let s3_inputs = policy.s3_inputs.as_ref().unwrap(); let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let on_behalf_authed = fetch_api_authed_from_permissioned_as( permissioned_as.clone(), @@ -4993,7 +5085,7 @@ async fn get_on_behalf_authed_from_app( let opt_authed = guest_caller_for_mode(opt_authed.clone(), policy.execution_mode(), path)?; let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let on_behalf_authed = fetch_api_authed_from_permissioned_as(permissioned_as, email, &w_id, &db, Some(username)) @@ -5504,7 +5596,45 @@ async fn app_load_csv_preview() -> Result<()> { )) } -fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { +/// The address to store beside the principal. Derived from it, never taken from the request, so +/// the stored copy can only ever agree with the principal — the drift it used to allow is what +/// this replaces. +/// +/// Written unconditionally, including for the versions that could derive it instead: a replica +/// predating that fallback fails outright when the key is absent, which would 400 every +/// anonymous, publisher and guest app for the length of a rolling deploy. The write is what +/// holds the key in place — see `docs/app-policy-email-removal.md`. +async fn stored_on_behalf_of_email(policy: &Policy, w_id: &str, db: &DB) -> Result> { + let Some(permissioned_as) = policy.on_behalf_of.as_deref() else { + return Ok(None); + }; + Ok(Some( + windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) + .await?, + )) +} + +/// The address to record in the `apps.on_behalf_of` audit entry: the one the app will run as, +/// when it is not the deployer's own. `None` when they match — a deployer handing an app their +/// own identity is not an on-behalf-of deploy. +/// +/// Reads the address `stored_on_behalf_of_email` just resolved rather than looking it up again, +/// so the audit row and the policy row can only ever name the same account. +fn audited_on_behalf_of(policy: &Policy, authed: &ApiAuthed) -> Option { + policy + .on_behalf_of_email + .as_deref() + .filter(|email| *email != authed.email) + .map(str::to_string) +} + +/// The identity an anonymous, publisher or guest execution runs as. +/// +/// `on_behalf_of_email` is optional: every write stores it, so it is present on anything this +/// release deployed, and it is only derived for a policy that predates that. Deriving is the +/// fallback rather than the rule so that removing the key later is a change of default, not a +/// change of behavior — see `docs/app-policy-email-removal.md`. +async fn get_on_behalf_of(policy: &Policy, w_id: &str, db: &DB) -> Result<(String, String)> { let permissioned_as = policy .on_behalf_of .as_ref() @@ -5515,16 +5645,15 @@ fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { ) })? .to_string(); - let email = policy - .on_behalf_of_email - .as_ref() - .ok_or_else(|| { - Error::BadRequest( - "on_behalf_of_email is missing in the app policy and is required for anonymous execution" - .to_string(), - ) - })? - .to_string(); + let email = match policy.on_behalf_of_email.as_deref() { + Some(email) => email.to_string(), + // Cached on purpose, up to one notify poll stale: the accepted dispatch case + // `get_email_from_permissioned_as` documents. + None => { + windmill_common::users::get_email_from_permissioned_as(&permissioned_as, w_id, db) + .await? + } + }; // Defence in depth against a policy that already carries a forged superadmin // sentinel (deployed before validation existed, or copied verbatim by a // workspace fork): the sentinels are internal-only and never a legitimate app @@ -5671,7 +5800,25 @@ async fn build_args( "email" => authed.as_ref().map(|a| serde_json::to_value(&a.email)), "workspace" => Some(serde_json::to_value(&w_id)), "groups" => authed.as_ref().map(|a| serde_json::to_value(&a.groups)), - "author" => Some(serde_json::to_value(&policy.on_behalf_of_email)), + // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. + "author" => { + let author = match ( + policy.on_behalf_of_email.as_deref(), + policy.on_behalf_of.as_deref(), + ) { + (Some(email), _) => Some(email.to_string()), + (None, Some(permissioned_as)) => Some( + windmill_common::users::get_email_from_permissioned_as( + permissioned_as, + w_id, + db, + ) + .await?, + ), + (None, None) => None, + }; + Some(serde_json::to_value(&author)) + } _ => { return Err(Error::BadRequest(format!( "context variable {} not allowed", @@ -5825,6 +5972,9 @@ mod embed_token_tests { "GET", ), ("/api/w/test/resources/list_search", "GET"), + // The metadata allowlist matches `resources/type/` by prefix, so a route + // added under it that is not a read must be denied by its method. + ("/api/w/test/resources/type/hub/pick/slack", "POST"), // Workspace-wide job enumeration/export must NOT be reachable — an app // reads only jobs it launched, by id (blocked via the app_embed sentinel). ("/api/w/test/jobs/list", "GET"), @@ -5918,6 +6068,10 @@ mod embed_token_tests { // the author declared it and the viewer consented. ("/api/w/test/resources/get_value/u/admin/r", "GET"), ("/api/w/test/variables/get_value/u/admin/v", "GET"), + // A chat UI's history for a chat-mode flow; RLS keeps it to the viewer's own. + ("/api/w/test/flow_conversations/list", "GET"), + ("/api/w/test/flow_conversations/some-uuid/messages", "GET"), + ("/api/w/test/flow_conversations/delete/some-uuid", "DELETE"), ]; for (path, method) in allowed { assert!( diff --git a/backend/windmill-api/src/csrf.rs b/backend/windmill-api/src/csrf.rs new file mode 100644 index 0000000000..4b2915f791 --- /dev/null +++ b/backend/windmill-api/src/csrf.rs @@ -0,0 +1,251 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::extract::FromRequestParts; +use axum::http::{header, request::Parts, Method}; +use axum::Extension; +use url::Url; +use windmill_common::error::Error; +use windmill_common::users::COOKIE_NAME; + +use crate::triggers::trigger_helpers::RunnableId; + +/// Whether a request is a cross-site GET authenticating on the session cookie alone. A GET +/// handler that runs a script by path resolves it through [`Self::script_runnable`], which +/// refuses a Hub script on such a request. +/// +/// The cookie is `SameSite=Lax`, so browsers attach it to cross-site top-level GET +/// navigations. A `hub/` path runs any public Hub script, and an argument written +/// `$var:` or `$res:` is resolved as the caller before the script sees it: such a +/// GET lets any page pick a generic Hub script and hand it the victim's secrets, which the job +/// can then send anywhere. +/// +/// Workspace scripts and flows are not refused, by choice, so that GET links to them keep +/// working. That is a scope decision, not a safety property: they still take attacker-chosen +/// arguments, `$var:` and `$res:` included, resolved as the victim. What bounds the exposure +/// is that the attacker needs a runnable path and can only run code the workspace deployed. +/// +/// The cookie is the only ambient credential. A bearer header is explicit, and so is the +/// `token` query parameter the webhook URLs carry — a cross-origin `EventSource` has no other +/// way to authenticate, since it cannot set headers. The checks run in `extract_token`'s +/// order, header before cookie, because that is the order it resolves them in: a request +/// carrying both a cookie and `token=` authenticates on the cookie and is therefore still +/// ambient, which is also why a valid `token=` link opened cross-site while signed in is +/// refused. +pub struct CrossSiteGetGuard(Option); + +impl CrossSiteGetGuard { + pub fn script_runnable(&self, script_path: &str) -> windmill_common::error::Result { + let runnable_id = RunnableId::from_script_path(script_path); + let (Some(signal), RunnableId::HubScript(_)) = (&self.0, &runnable_id) else { + return Ok(runnable_id); + }; + // The `Referer` leg is the one that can misfire, on a request that really was + // same-host: it compares against the hosts the backend can see, and a proxy that + // rewrites `Host` without setting `X-Forwarded-Host` leaves none of them matching + // what the browser addressed. Name the comparison so that shows up as a + // misconfiguration rather than as an unexplained 403. + if let CrossSite::RefererMismatch { referer, instance_hosts } = signal { + tracing::warn!( + referer_host = %referer, + ?instance_hosts, + "refusing a cross-site GET Hub script run inferred from Referer; if the request \ + was same-host, set `X-Forwarded-Host` on the proxy or configure `BASE_URL`" + ); + } + Err(Error::PermissionDenied( + "a cross-site GET request cannot run a Hub script with the session cookie, which takes \ + precedence over a `token` query parameter: pass the token in the `Authorization` \ + header, or open the link from the instance itself or from a browser with no Windmill \ + session" + .to_string(), + )) + } +} + +impl FromRequestParts for CrossSiteGetGuard { + type Rejection = std::convert::Infallible; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method != Method::GET { + return Ok(CrossSiteGetGuard(None)); + } + let Some(signal) = cross_site_signal(parts) else { + return Ok(CrossSiteGetGuard(None)); + }; + + let has_bearer = parts + .headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.starts_with("Bearer ")); + if has_bearer { + return Ok(CrossSiteGetGuard(None)); + } + + let has_session_cookie = + Extension::::from_request_parts(parts, state) + .await + .is_ok_and(|Extension(cookies)| cookies.get(COOKIE_NAME).is_some()); + Ok(CrossSiteGetGuard(has_session_cookie.then_some(signal))) + } +} + +enum CrossSite { + Declared, + RefererMismatch { referer: String, instance_hosts: Vec }, +} + +fn cross_site_signal(parts: &Parts) -> Option { + if let Some(site) = parts.headers.get("sec-fetch-site") { + return site + .as_bytes() + .eq_ignore_ascii_case(b"cross-site") + .then_some(CrossSite::Declared); + } + // Fetch Metadata rides only on potentially trustworthy URLs, so an instance served + // over plain http never receives `Sec-Fetch-Site` (nor does Safari before 16.4) while + // the cookie, not being `Secure` there either, still arrives. `Referer` is the only + // other thing a top-level GET navigation carries — `Origin` is not sent on one — so it + // is all that is left there, and it is weak: the default `strict-origin-when-cross- + // origin` policy already drops `Referer` on an https-to-http downgrade, so an https + // attacker page pointing a victim at a plain-http instance sends neither header. This + // leg catches an http-served attacker page and pre-16.4 Safari on https; the guard is + // load-bearing on https and best-effort at best on plain http. An absent `Referer` + // reads as not cross-site, matching how `Sec-Fetch-Site: none` (a bookmark, a typed + // URL) is treated. + let referer = referer_host(parts)?; + let instance_hosts: Vec = instance_hosts(parts).collect(); + (!instance_hosts + .iter() + .any(|host| host.eq_ignore_ascii_case(&referer))) + .then_some(CrossSite::RefererMismatch { referer, instance_hosts }) +} + +/// Every host a legitimate same-host request can name. `Host` alone is not enough: a +/// reverse proxy that forwards without preserving it (nginx `proxy_pass` with no +/// `proxy_set_header Host $host`) hands the backend the upstream's name, which no browser +/// `Referer` will ever match. None of these is browser-settable on a navigation — a +/// navigation carries no custom headers, and `BASE_URL` is instance config — so widening +/// the accepted set costs nothing. +fn instance_hosts(parts: &Parts) -> impl Iterator { + let base_url = windmill_common::BASE_URL.load(); + [ + request_host(parts), + header_host(parts, "x-forwarded-host"), + Url::parse(base_url.as_str()) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)), + ] + .into_iter() + .flatten() +} + +fn referer_host(parts: &Parts) -> Option { + let referer = parts.headers.get(header::REFERER)?.to_str().ok()?; + Url::parse(referer).ok()?.host_str().map(str::to_owned) +} + +fn request_host(parts: &Parts) -> Option { + if let Some(host) = parts.uri.host() { + return Some(host.to_owned()); + } + header_host(parts, header::HOST) +} + +fn header_host(parts: &Parts, name: impl header::AsHeaderName) -> Option { + host_of(parts.headers.get(name)?.to_str().ok()?) +} + +/// The host in a `Host`-shaped header value: `host[:port]`, where `host` may be a bracketed +/// IPv6 literal, and where a chain of proxies appends to `X-Forwarded-Host` so only the +/// first entry is the one the browser addressed. The port is split off by the URL parser +/// rather than by hand-rolling the bracket rules. +fn host_of(value: &str) -> Option { + let host = value.split(',').next()?.trim(); + Url::parse(&format!("http://{host}")) + .ok()? + .host_str() + .map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::{cross_site_signal, host_of, CrossSite}; + use axum::http::{request::Parts, Request}; + + fn parts(headers: &[(&str, &str)]) -> Parts { + let mut req = Request::get("/api/w/ws/jobs/run_wait_result/p/hub/1/x"); + for (name, value) in headers { + req = req.header(*name, *value); + } + req.body(()).unwrap().into_parts().0 + } + + #[test] + fn sec_fetch_site_decides_when_present() { + let declared = |site| cross_site_signal(&parts(&[("sec-fetch-site", site)])); + assert!(matches!(declared("cross-site"), Some(CrossSite::Declared))); + for site in ["same-origin", "same-site", "none"] { + assert!(declared(site).is_none(), "{site} is not cross-site"); + } + // The header outranks a `Referer` that disagrees with it. + let with_referer = parts(&[ + ("sec-fetch-site", "same-origin"), + ("host", "windmill.example"), + ("referer", "https://attacker.example/page"), + ]); + assert!(cross_site_signal(&with_referer).is_none()); + } + + #[test] + fn referer_stands_in_when_sec_fetch_site_is_absent() { + let signal = |headers: &[(&str, &str)]| cross_site_signal(&parts(headers)); + assert!(matches!( + signal(&[ + ("host", "windmill.example"), + ("referer", "https://attacker.example/p") + ]), + Some(CrossSite::RefererMismatch { .. }) + )); + // Ports differ between the frontend and the API, and do not make a request cross-site. + assert!(signal(&[ + ("host", "windmill.example:8000"), + ("referer", "http://windmill.example:3000/apps"), + ]) + .is_none()); + // A proxy that rewrote `Host` but forwarded the public name. + assert!(signal(&[ + ("host", "windmill-server.internal"), + ("x-forwarded-host", "windmill.example"), + ("referer", "https://windmill.example/apps"), + ]) + .is_none()); + assert!(signal(&[("host", "windmill.example")]).is_none()); + } + + #[test] + fn host_of_strips_port_brackets_and_proxy_chain() { + assert_eq!(host_of("windmill.example"), Some("windmill.example".into())); + assert_eq!( + host_of("windmill.example:8000"), + Some("windmill.example".into()) + ); + assert_eq!(host_of("[::1]:8000"), Some("[::1]".into())); + assert_eq!(host_of("[::1]"), Some("[::1]".into())); + assert_eq!( + host_of("windmill.example, proxy.internal"), + Some("windmill.example".into()) + ); + assert_eq!(host_of(""), None); + assert_eq!(host_of("not a host"), None); + } +} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index d781e229d2..e8a6858db7 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -108,6 +108,9 @@ lazy_static::lazy_static! { (20260826214706, include_str!( "../../migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql" ).replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260909163047, include_str!( + "../../migrations/20260909163047_workspace_delete_cascade_indexes.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } diff --git a/backend/windmill-api/src/git_sync_oss.rs b/backend/windmill-api/src/git_sync_oss.rs index 0451d88699..d1653d9aa9 100644 --- a/backend/windmill-api/src/git_sync_oss.rs +++ b/backend/windmill-api/src/git_sync_oss.rs @@ -10,6 +10,11 @@ pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] +pub fn workspaced_git_sync_service() -> Router { + Router::new() +} + #[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index ccb530ce20..633ad88a34 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -6,13 +6,14 @@ use axum::{ http::{request::Parts, StatusCode}, response::{IntoResponse, Response}, routing::{get, post}, - Router, + Extension, Router, }; use serde::{Deserialize, Deserializer, Serialize}; use windmill_common::{ error::{to_anyhow, Error}, + global_settings::{load_value_from_global_settings, DISABLE_HUB_SETTING}, utils::require_admin, - HUB_BASE_URL, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, }; pub fn workspaced_service() -> Router { @@ -48,6 +49,7 @@ pub fn workspaced_service() -> Router { post(discard_project_update), ) .route("/project", get(get_project_by_source)) + .route("/projects", get(list_projects)) } #[derive(Deserialize)] @@ -548,6 +550,84 @@ async fn get_project_by_source(ctx: HubPublishCtx) -> Result bool { + fn host_of(url: &str) -> Option { + let parsed = url::Url::parse(url.trim()).ok()?; + if !matches!(parsed.scheme(), "http" | "https") { + return None; + } + Some( + parsed + .host_str()? + .trim_end_matches('.') + .to_ascii_lowercase(), + ) + } + match (host_of(hub), host_of(DEFAULT_HUB_BASE_URL)) { + (Some(host), Some(default_host)) => host == default_host, + _ => false, + } +} + +// The hub's project catalogue. Read by any workspace member rather than through +// `HubPublishCtx`, which requires an admin: nothing here is workspace-scoped or +// publishing-related. It exists at all because the hub's listing endpoint sends no +// CORS header, so the browser cannot read it directly the way it reads a single +// project. `accept: application/json` is what makes the hub answer with JSON. +// +// The caller's token is sent only to a hub this instance was pointed at deliberately. +// Every other route here is admin-only; this one is not, so forwarding a member's +// bearer token to `hub.windmill.dev` would put a credential replayable against this +// instance on a host outside it — for a listing that needs no credential at all. +async fn list_projects( + _authed: ApiAuthed, + Extension(db): Extension, + Tokened { token }: Tokened, +) -> Result { + // `disable_hub` turns the hub off for a closed instance, and this handler makes an + // outbound request. The frontend hides its entry points on the same setting, but that + // is presentation: an authenticated member can call this route directly, so the refusal + // has to live here. + let disabled = load_value_from_global_settings(&db, DISABLE_HUB_SETTING) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if disabled { + return Err(Error::BadRequest( + "The hub is disabled on this instance".to_string(), + )); + } + + let hub = (**HUB_BASE_URL.load()).clone(); + let url = format!("{}/projects", hub); + let mut req = HTTP_CLIENT.get(&url).header("accept", "application/json"); + if !is_public_hub(&hub) { + req = req.bearer_auth(&token); + } + let res = req + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} + async fn submit_project( ctx: HubPublishCtx, Path((_workspace, slug)): Path<(String, ProjectSlug)>, @@ -645,3 +725,42 @@ async fn forward_to_hub( Ok((status, text)) } + +#[cfg(test)] +mod tests { + use super::is_public_hub; + + #[test] + fn public_hub_recognized_in_every_spelling() { + // The predicate decides whether a workspace member's bearer token leaves the + // instance, so both directions matter: a miss on the public hub sends the token + // to windmill.dev, and a false match withholds it from a private hub that needs it. + // Every spelling here is one `hub_base_url` can hold and `reqwest` will still send. + for hub in [ + "https://hub.windmill.dev", + "http://hub.windmill.dev/", + "HTTPS://hub.windmill.dev", + "https://HUB.WINDMILL.DEV", + "https://hub.windmill.dev:443", + "https://hub.windmill.dev.", + "https://hub.windmill.dev/some/path", + " https://hub.windmill.dev ", + ] { + assert!(is_public_hub(hub), "{hub} should be the public hub"); + } + for hub in [ + "https://hub.internal.example", + "https://hub.windmill.dev.evil.example", + "https://windmill.dev", + // The host is what the request goes to, whatever precedes the `@`. + "https://hub.windmill.dev@hub.internal.example", + // Unparseable, or not a scheme a request can be built from. Grouped with the + // private hubs because the caller then attaches the token, which is harmless here: + // `reqwest` rejects the same value before opening a connection. + "hub.windmill.dev", + "ftp://hub.windmill.dev", + ] { + assert!(!is_public_hub(hub), "{hub} should not be the public hub"); + } + } +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index dbf33cdddc..84968ebbcc 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -73,6 +73,7 @@ use crate::{ args::{self, RawWebhookArgs}, auth::{OptTokened, Tokened}, concurrency_groups::join_concurrency_key, + csrf::CrossSiteGetGuard, db::{ApiAuthed, DB}, triggers::trigger_helpers::RunnableId, users::{ @@ -106,7 +107,10 @@ use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error}, flow_status::{Approval, ApprovalConditions, FlowStatus, FlowStatusModule}, - flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, + flows::{ + add_virtual_items_if_necessary, resolve_maybe_value, ApprovalSkin, FlowModule, FlowValue, + Suspend, + }, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, query_builders, @@ -139,6 +143,7 @@ pub fn workspaced_service() -> Router { .route("/run_progress/{id}", get(get_run_progress)) .route("/run_assets/{id}", get(list_run_assets)) .route("/dbt_graph/{id}", get(get_dbt_run_graph)) + .route("/dbt_column_lineage/{id}", get(get_dbt_run_column_lineage)) .route("/dbt_resumable/{id}", get(get_dbt_resumable)) .route( "/dbt_resumable_script/p/{*script_path}", @@ -891,21 +896,27 @@ struct AssetProgress { error: Option, } -/// The asset graph as one run saw it. Pinning to a job needs the full job-read -/// contract, so it lives on `require_job_read_access` here rather than as a -/// parameter on `/assets/graph`. See docs/dbt-runtime.md. -async fn get_dbt_run_graph( - authed: ApiAuthed, - OptViewToken(view_token): OptViewToken, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Query(q): Query, -) -> error::JsonResult { +/// Which project version a dbt view pins to for this job, once the caller has +/// been shown to be entitled to it. +/// +/// `Ok(None)` is "answer unpinned", not a refusal: a job that stored no graph of +/// its own — and one that has aged out of retention — is served the deployed +/// version rather than an error, so a run page keeps drawing after the run is +/// gone. Pinning needs the full job-read contract, which is why it lives on +/// `require_job_read_access` here rather than as a parameter on `/assets/*`. +/// See docs/dbt-runtime.md. +async fn dbt_pinned_run( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + job_id: Uuid, + view_token: Option<&str>, +) -> error::Result> { // The scope domain comes from the URL segment, so `/jobs` asks a scoped token // for `jobs:read` alone while the body returned is asset data. Both are // required: the job gate below reaches this run, this reaches assets at all. - check_scopes(&authed, || "assets:read".to_string())?; + check_scopes(authed, || "assets:read".to_string())?; let job = sqlx::query!( r#"SELECT created_by, runnable_path, CASE WHEN kind = 'script' THEN runnable_id END AS script_hash, @@ -918,42 +929,70 @@ async fn get_dbt_run_graph( AND g.script_hash IS NULL) AS "editor_graph!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, job_id, - &w_id + w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await?; - // No such job: answer the unpinned graph rather than 404, so a run page whose - // job has aged out of retention still draws the deployed version instead of - // an error. Reachable only with `assets:read`, which is exactly what - // `/assets/graph` would have cost for the same answer. + // Unpinned rather than 404 for a job that is gone. Reachable only with + // `assets:read`, which is exactly what the unpinned route would have cost + // for the same answer. let Some(job) = job else { - return windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, None).await; + return Ok(None); }; require_job_read_access( - &db, - &user_db, - &authed, - &w_id, + db, + user_db, + authed, + w_id, &job_id, &job.created_by, - view_token.as_deref(), + view_token, ) .await?; // A preview or flow job names no deployed version, so there is usually no // graph to pin to and the workspace one answers. The exception is a job that // parsed one itself, which is what the dbt editor's refresh is: its graph // belongs to that job alone and nothing else can reach it. - let pinned = job + Ok(job .runnable_path .filter(|_| job.script_hash.is_some() || job.editor_graph) .map(|path| windmill_api_assets::PinnedRun { job_id, script_path: path, script_hash: job.script_hash, - }); + })) +} + +/// The asset graph as one run saw it. +async fn get_dbt_run_graph( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(q): Query, +) -> error::JsonResult { + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, pinned).await } +/// The column lineage a set of relations sits in as one run saw it — the same +/// pin as `get_dbt_run_graph`, for the trace drawn beside a node of that graph. +async fn get_dbt_run_column_lineage( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(pairs): Query>, +) -> error::JsonResult { + let q = windmill_api_assets::ColumnLineageQuery::from_query_pairs(pairs)?; + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; + windmill_api_assets::dbt_column_lineage_for(&authed, &w_id, user_db, q, pinned).await +} + /// Whether a `dbt retry` submitted by this caller would resume THIS run. /// /// One failure is saved per script per execution principal, so a page showing an @@ -4381,6 +4420,15 @@ async fn count_completed_jobs( )) } +lazy_static::lazy_static! { + /// 0 keeps the connection-wide statement_timeout. + static ref LIST_JOBS_STATEMENT_TIMEOUT_SECS: u64 = + std::env::var("LIST_JOBS_STATEMENT_TIMEOUT_SECS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(30); +} + async fn list_jobs( authed: ApiAuthed, Extension(user_db): Extension, @@ -4498,10 +4546,32 @@ async fn list_jobs( // tracing::info!("sql: {}", &sql); let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + // A client that gives up does not cancel its query, so without this bound every retry of a + // slow filter stacks another scan running until the connection-wide 5min timeout. + let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; + if timeout_secs > 0 { + sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'")) + .execute(&mut *tx) + .await?; + } + let jobs: Vec = sqlx::query_as(&sql) .fetch_all(&mut *tx) .warn_after_seconds_with_sql(5, format!("list_jobs: {}", sql)) - .await?; + .await + .map_err(|e| match e { + sqlx::Error::Database(ref db_err) + if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") => + { + Error::Generic( + StatusCode::BAD_REQUEST, + format!( + "Listing jobs took more than {timeout_secs}s and was stopped. Set a start date or narrow the filters." + ), + ) + } + e => e.into(), + })?; tx.commit().await?; Ok(Json(jobs.into_iter().map(From::from).collect())) @@ -4791,6 +4861,11 @@ struct ApprovalInfo { user_auth_required: bool, #[serde(skip_serializing_if = "Option::is_none")] hide_cancel: Option, + skin: ApprovalSkin, + #[serde(skip_serializing_if = "Option::is_none")] + step_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_summary: Option, approvers: Vec, /// Share-read-link token for the flow, minted only for callers allowed to view this /// approval. Lets an authenticated workspace-member approver open the run details of @@ -4844,6 +4919,48 @@ fn can_approve_step( } } +/// The latest approval step the run has passed: a step before the current `step` that ran +/// rather than being skipped. Steps from `step` on don't count, because while an approval is +/// pending the step after it already holds the `WaitingForEvents` status. +fn last_reached_approval_step<'a>( + flow: &'a FlowValue, + status: &FlowStatus, +) -> Option<&'a FlowModule> { + flow.modules + .iter() + .zip(status.modules.iter()) + .take(usize::try_from(status.step).unwrap_or(0)) + .rev() + .filter(|(_, m)| matches!(m, FlowStatusModule::Success { skipped: false, .. })) + .map(|(module, _)| module) + .find(|module| module.suspend.is_some()) +} + +/// The approval conditions a step's own settings give, as the worker records them when the step +/// suspends. The worker drops them from the run once the step is approved, so a run that has +/// moved on is gated by these. Groups computed by an expression can't be re-evaluated outside +/// the run, so such a step falls back to any signed-in user. +fn approval_conditions_from_settings(suspend: &Suspend) -> Option { + let user_auth_required = suspend.user_auth_required.unwrap_or(false); + let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false); + if !user_auth_required && !self_approval_disabled { + return None; + } + let user_groups_required = match &suspend.user_groups_required { + Some(InputTransform::Static { value }) if user_auth_required => { + serde_json::from_str(value.get()).unwrap_or_default() + } + _ => vec![], + }; + Some(ApprovalConditions { user_auth_required, user_groups_required, self_approval_disabled }) +} + +/// How the approval step presents itself on the approval page. +struct ApprovalStepView { + skin: ApprovalSkin, + summary: Option, +} + async fn get_approval_info( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -4872,13 +4989,33 @@ async fn get_approval_info( script_path: Option, email: String, flow_status: Option, - workflow_as_code_status: Option, + // `v2_job_status` only holds a run that hasn't finished, so the fields below also read + // the completed run's status: a finished run's page keeps its skin and, for workflows as + // code, its description, still gated by the approval conditions the run had. + completed_flow_status: Option, + is_wac: bool, + wac_approval: Option, + approval_conditions: Option, + flow_summary: Option, } let row = sqlx::query_as::<_, ApprovalJobRow>( "SELECT j.id, j.runnable_path as script_path, j.permissioned_as_email as email, - s.flow_status, s.workflow_as_code_status + s.flow_status, + c.flow_status AS completed_flow_status, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status) IS NOT NULL + AS is_wac, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status)->'_approval' + AS wac_approval, + COALESCE(s.flow_status, c.flow_status)->'approval_conditions' + AS approval_conditions, + NULLIF(COALESCE(f.summary, sc.summary), '') AS flow_summary FROM v2_job j LEFT JOIN v2_job_status s ON s.id = j.id + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN flow f + ON j.kind = 'flow' AND f.workspace_id = j.workspace_id AND f.path = j.runnable_path + LEFT JOIN script sc + ON j.kind = 'script' AND sc.workspace_id = j.workspace_id AND sc.hash = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", ) .bind(&job_id) @@ -4887,31 +5024,31 @@ async fn get_approval_info( .await? .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; - let is_wac = row.workflow_as_code_status.is_some(); + let is_wac = row.is_wac; + let run_ac = row + .approval_conditions + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()); // Extract approval info based on WAC vs classic flow - let (form_schema, description, default_args, enums, approval_conditions, hide_cancel) = + let (form_schema, description, default_args, enums, approval_conditions, hide_cancel, step) = if is_wac { - let approval_meta = row - .workflow_as_code_status - .as_ref() - .and_then(|v| v.get("_approval")); + let approval_meta = row.wac_approval.as_ref(); let form = approval_meta.and_then(|m| m.get("form").cloned()); let default_args = approval_meta.and_then(|m| m.get("default_args").cloned()); let enums = approval_meta.and_then(|m| m.get("enums").cloned()); let description = approval_meta.and_then(|m| m.get("description").cloned()); - let ac = row - .flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()); - (form, description, default_args, enums, ac, None) + let skin = approval_meta + .and_then(|m| m.get("skin")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .unwrap_or_default(); + let step = Some(ApprovalStepView { skin, summary: None }); + (form, description, default_args, enums, run_ac, None, step) } else { let fs = row .flow_status .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()); - let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); // For classic flows, form/description come from the flow definition and step result let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); @@ -4971,6 +5108,28 @@ async fn get_approval_info( .and_then(|s| s.resume_form.as_ref()) .map(|rf| serde_json::json!(rf)); let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); + let completed_fs = row + .completed_flow_status + .as_ref() + .filter(|_| fs.is_none()) + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let approval_module = raw_flow + .as_ref() + .zip(fs.as_ref().or(completed_fs.as_ref())) + .and_then(|(flow, status)| last_reached_approval_step(flow, status)); + let ac = run_ac.or_else(|| { + approval_module + .and_then(|module| module.suspend.as_ref()) + .and_then(approval_conditions_from_settings) + }); + let step = approval_module.map(|module| ApprovalStepView { + skin: module + .suspend + .as_ref() + .and_then(|s| s.skin) + .unwrap_or_default(), + summary: module.summary.clone().filter(|s| !s.trim().is_empty()), + }); // Fetch description, default_args, and enums from the step's completed job result let step_job_id = fs @@ -4994,9 +5153,12 @@ async fn get_approval_info( (None, None, None) }; - (form, desc, default_args, enums, ac, hc) + (form, desc, default_args, enums, ac, hc, step) }; + let skin = step.as_ref().map(|s| s.skin).unwrap_or_default(); + let step_summary = step.and_then(|s| s.summary); + let user_auth_required = approval_conditions .as_ref() .map(|ac| ac.user_auth_required) @@ -5026,6 +5188,9 @@ async fn get_approval_info( can_approve: false, user_auth_required, hide_cancel: None, + skin, + step_summary: None, + flow_summary: None, approvers: vec![], view_token: None, })); @@ -5061,6 +5226,9 @@ async fn get_approval_info( can_approve, user_auth_required, hide_cancel, + skin, + step_summary, + flow_summary: row.flow_summary, approvers, view_token, })) @@ -6476,7 +6644,7 @@ pub async fn run_flow_by_path( Query(run_query): Query, args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let (args, trigger_metadata) = get_args_and_trigger_metadata( + let (args, trigger_metadata) = match get_args_and_trigger_metadata( &db, &authed, RunnableId::from_flow_path(flow_path.to_path()), @@ -6484,7 +6652,15 @@ pub async fn run_flow_by_path( &w_id, args, ) - .await?; + .await? + { + WebhookRun::Run(args, trigger_metadata) => (args, trigger_metadata), + // 200 rather than an error: services drop a webhook that keeps failing, and disabling a + // trigger in Windmill must not cost it its registration. + WebhookRun::TriggerDisabled => { + return Ok((StatusCode::OK, NATIVE_TRIGGER_DISABLED_MSG.to_string())) + } + }; let (uuid, _, _, _) = push_flow_job_by_path_into_queue( authed, @@ -6909,7 +7085,7 @@ pub async fn run_script_by_path( Query(run_query): Query, args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let (args, trigger_metadata) = get_args_and_trigger_metadata( + let (args, trigger_metadata) = match get_args_and_trigger_metadata( &db, &authed, RunnableId::from_script_path(script_path.to_path()), @@ -6917,7 +7093,13 @@ pub async fn run_script_by_path( &w_id, args, ) - .await?; + .await? + { + WebhookRun::Run(args, trigger_metadata) => (args, trigger_metadata), + WebhookRun::TriggerDisabled => { + return Ok((StatusCode::OK, NATIVE_TRIGGER_DISABLED_MSG.to_string())) + } + }; let (uuid, _, _) = push_script_job_by_path_into_queue( authed, @@ -6935,6 +7117,16 @@ pub async fn run_script_by_path( Ok((StatusCode::CREATED, uuid.to_string())) } +/// What a webhook delivery resolved to: the arguments to run with, or nothing to run. +pub enum WebhookRun { + Run(PushArgsOwned, Option), + /// The native trigger this delivery belongs to is disabled. + TriggerDisabled, +} + +const NATIVE_TRIGGER_DISABLED_MSG: &str = + "This trigger is disabled in Windmill, so no job was created"; + #[allow(unused)] pub async fn get_args_and_trigger_metadata( db: &DB, @@ -6943,14 +7135,21 @@ pub async fn get_args_and_trigger_metadata( run_query: &RunJobQuery, w_id: &str, args: RawWebhookArgs, -) -> error::Result<(PushArgsOwned, Option)> { +) -> error::Result { use windmill_common::triggers::TriggerMetadata; // Build trigger metadata if this is a native trigger request #[cfg(feature = "native_trigger")] let (trigger_metadata, native_args) = if let Some(service_name_str) = &run_query.service_name { - use crate::native_triggers::{prepare_native_trigger_args, ServiceName}; + use crate::native_triggers::{ + native_trigger_is_enabled, prepare_native_trigger_args, ServiceName, + }; let service_name = ServiceName::try_from(service_name_str.to_owned())?; + if let Some(external_id) = run_query.trigger_external_id.as_deref() { + if !native_trigger_is_enabled(db, w_id, service_name, external_id).await? { + return Ok(WebhookRun::TriggerDisabled); + } + } let metadata = Some(TriggerMetadata::new( run_query.trigger_external_id.clone(), service_name.as_job_trigger_kind(), @@ -6986,7 +7185,7 @@ pub async fn get_args_and_trigger_metadata( .await? }; - Ok((args, trigger_metadata)) + Ok(WebhookRun::Run(args, trigger_metadata)) } #[derive(Deserialize)] @@ -7381,6 +7580,7 @@ async fn log_job_view( } pub async fn run_wait_result_job_by_path_get( + cross_site: CrossSiteGetGuard, method: hyper::http::Method, authed: ApiAuthed, Extension(user_db): Extension, @@ -7393,6 +7593,7 @@ pub async fn run_wait_result_job_by_path_get( check_license_key_valid().await?; let script_path = script_path.to_path(); + let runnable_id = cross_site.script_runnable(script_path)?; check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; if method == http::Method::HEAD { @@ -7405,12 +7606,7 @@ pub async fn run_wait_result_job_by_path_get( args.body = args::Body::HashMap(payload_as_args); let args = args - .to_args_from_runnable( - &db, - &w_id, - RunnableId::from_script_path(script_path), - run_query.skip_preprocessor, - ) + .to_args_from_runnable(&db, &w_id, runnable_id, run_query.skip_preprocessor) .await?; check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; @@ -7829,6 +8025,7 @@ pub async fn stream_flow_by_version( } pub async fn stream_script_by_path( + cross_site: CrossSiteGetGuard, authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, @@ -7837,12 +8034,13 @@ pub async fn stream_script_by_path( method: hyper::http::Method, args: RawWebhookArgs, ) -> error::Result { + let runnable_id = cross_site.script_runnable(script_path.to_path())?; stream_job( authed, db, user_db, w_id, - RunnableId::from_script_path(script_path.to_path()), + runnable_id, args, run_query, method == http::Method::GET, @@ -11811,4 +12009,58 @@ mod approval_view_gate_tests { "trigger@example.com" )); } + + #[test] + fn approval_step_is_the_last_one_passed() { + let flow: FlowValue = serde_json::from_value(serde_json::json!({ "modules": [ + { "id": "a", "value": { "type": "identity" }, "suspend": {} }, + { "id": "b", "value": { "type": "identity" }, "suspend": {} }, + { "id": "c", "value": { "type": "identity" } } + ]})) + .unwrap(); + let step_at = |step: i32, types: [(&str, bool); 3]| { + let mut status = FlowStatus::new(&flow); + status.step = step; + status.modules = ["a", "b", "c"] + .into_iter() + .zip(types) + .map(|(id, (kind, skipped))| { + serde_json::from_value(serde_json::json!({ + "type": kind, "id": id, "job": Uuid::nil(), "count": 1, + "failed_retries": [], "skipped": skipped + })) + .unwrap() + }) + .collect(); + last_reached_approval_step(&flow, &status).map(|module| module.id.clone()) + }; + let waiting = ("WaitingForEvents", false); + let pending = ("WaitingForPriorSteps", false); + let ran = ("Success", false); + let skipped = ("Success", true); + // Awaiting a's approval: b, itself an approval step, already holds `WaitingForEvents`. + assert_eq!(step_at(1, [ran, waiting, pending]).as_deref(), Some("a")); + assert_eq!(step_at(2, [ran, ran, waiting]).as_deref(), Some("b")); + assert_eq!(step_at(3, [ran, skipped, ran]).as_deref(), Some("a")); + assert_eq!(step_at(0, [pending, pending, pending]), None); + } + + #[test] + fn approved_step_stays_gated_by_its_settings() { + let from_settings = |suspend: serde_json::Value| { + approval_conditions_from_settings(&serde_json::from_value(suspend).unwrap()) + }; + let login = from_settings(serde_json::json!({ + "user_auth_required": true, + "user_groups_required": { "type": "static", "value": ["approvers"] } + })); + assert!(!can_view( + &None, + &login, + Some("f/team/flow"), + "trigger@example.com" + )); + assert_eq!(login.unwrap().user_groups_required, ["approvers"]); + assert!(from_settings(serde_json::json!({})).is_none()); + } } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 4ba60428c9..b70d80140a 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -69,6 +69,11 @@ mod ai; #[cfg(feature = "private")] mod ai_free_tier_ee; mod ai_free_tier_oss; +#[cfg(feature = "parquet")] +mod ai_sessions; +#[cfg(feature = "parquet")] +pub use ai_sessions::sweep_expired_ai_session_backups; +mod ai_shared_artifacts; mod apps; mod apps_raw_bundle; pub use apps::invalidate_app_policy_cache; @@ -80,6 +85,7 @@ pub mod azure_proxy_ee; mod azure_proxy_oss; mod capture; mod concurrency_groups; +mod csrf; mod db; mod db_health; mod dbt; @@ -658,9 +664,13 @@ pub async fn run_server( "/workspace_dependencies", workspace_dependencies::workspaced_service(), ) + // CORS so a chat UI on another origin (an external site, or + // a sandboxed raw app with its frontend SDK token) can read + // its conversation history. Bearer-only, like variables. .nest( "/flow_conversations", - windmill_api_flow_conversations::workspaced_service(), + windmill_api_flow_conversations::workspaced_service() + .layer(cors.clone()), ) // CORS so an opaque-origin app iframe (WIN-2006 embed, // no separate domain) can read folders/listnames with a @@ -959,6 +969,15 @@ pub async fn run_server( #[cfg(not(feature = "enterprise"))] Router::new() }) + .nest("/w/{workspace_id}/git_sync", { + #[cfg(feature = "enterprise")] + { + git_sync_oss::workspaced_git_sync_service() + } + + #[cfg(not(feature = "enterprise"))] + Router::new() + }) .nest( "/w/{workspace_id}/resources_u", public_service().layer(cors.clone()), diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index aa2415f664..ffefaba2dd 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1265,13 +1265,15 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1282,7 +1284,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } } @@ -1380,13 +1382,15 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1397,7 +1401,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } }, diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 2998faa7f3..26eb06c8e8 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -128,6 +128,17 @@ struct WorkspaceReassignment { // ---- Preview helpers ---- +/// The principal a departing member's runnables run as, or `None` when none of them are theirs +/// to hand over. `usr.username` is constrained to `[\w-]+`, so a member is `u/{username}`, except +/// a legacy `group-*` username, which canonicalizes to the group it names: what runs under that +/// principal runs as the group, which outlives the member. `None` binds NULL, which the +/// `on_behalf_of = $n` queries then match nowhere. +fn departing_principal(username: &str) -> Option { + use windmill_common::users::{username_to_permissioned_as, PERMISSIONED_AS_GROUP_PREFIX}; + let principal = username_to_permissioned_as(username); + (!principal.starts_with(PERMISSIONED_AS_GROUP_PREFIX)).then_some(principal) +} + async fn get_offboard_preview( db: impl sqlx::PgExecutor<'_> + Copy, w_id: &str, @@ -136,10 +147,8 @@ async fn get_offboard_preview( ) -> Result { let user_prefix = format!("u/{}/%", username); let user_owner = format!("u/{}", username); - // Same form the mutation reassigns, so preview and execution cannot disagree. `usr.username` - // is constrained to `[\w-]+`, so a member is always named `u/{username}` — the address form a - // principal can also take names an account with no `usr` row, which is nobody offboardable. - let departing = windmill_common::users::username_to_permissioned_as(username); + // Same form the mutation reassigns, so preview and execution cannot disagree. + let departing = departing_principal(username); // ---- Owned objects (under u/{username}/) ---- let scripts = sqlx::query_scalar!( @@ -244,17 +253,17 @@ async fn get_offboard_preview( // ---- Operator references (not under user's path) ---- let obo_scripts = sqlx::query_scalar!( "SELECT path FROM script WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived AND NOT deleted", - &departing, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_flows = sqlx::query_scalar!( "SELECT path FROM flow WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived", - &departing, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_apps = sqlx::query_scalar!( "SELECT path FROM app WHERE policy->>'on_behalf_of' = $1 AND NOT path LIKE $2 AND workspace_id = $3", - &user_owner, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_schedules = sqlx::query_scalar!( @@ -831,7 +840,7 @@ async fn offboard_user_from_workspace<'c>( new_permissioned_as: &str, ) -> Result { let new_prefix = reassign_to.to_string(); - let departing = windmill_common::users::username_to_permissioned_as(username); + let departing = departing_principal(username); // The app policy stores an address beside its principal, and script/flow keep one for the // workers that still read it, so the replacement's is resolved here. @@ -871,7 +880,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( "UPDATE script SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -912,7 +921,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( "UPDATE flow SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -925,7 +934,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( r#"UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of'], to_jsonb($1::text)), ARRAY['on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('script', 'flow') AND value->>'on_behalf_of' = $2 AND workspace_id = $3"#, new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -951,9 +960,21 @@ async fn offboard_user_from_workspace<'c>( "UPDATE app SET policy = jsonb_set( jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)), ARRAY['on_behalf_of_email'], to_jsonb($4::text) - ) WHERE policy->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + ) WHERE policy->>'on_behalf_of' = $2 AND workspace_id = $3", &new_permissioned_as, - username, + departing.as_deref(), + w_id, + new_on_behalf_of_user_email + ) + .execute(&mut **tx) + .await?; + + // An app draft carries a copy of the deployed policy and is deployed from it, so it needs + // the same pair rewritten — the draft sweep above only covers scripts and flows. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2 AND workspace_id = $3"#, + new_permissioned_as, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 593ea3b298..204b30010d 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -13,19 +13,25 @@ use sha2::Sha256; use sqlx::types::Uuid; use std::collections::HashMap; use windmill_common::error::{to_anyhow, Error}; +use windmill_common::flows::ApprovalSkin; +use windmill_common::utils::truncate_with_ellipsis; use windmill_common::variables::{get_secret_value_as_admin, get_workspace_key}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; use crate::{ approvals::{ - extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, - MessageFormat, QueryButtonText, QueryDefaultArgsJson, QueryDynamicEnumJson, - QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, + extract_w_id_from_resume_url, get_approval_step_skin, handle_resume_action, + ApprovalFormDetails, FieldType, MessageFormat, QueryButtonText, QueryDefaultArgsJson, + QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, }, auth::OptTokened, }; +// Slack rejects a button value over 2000 characters, and with it the whole post. The button value +// carries the message on to the modal, so the message is shortened to fit. +const SLACK_BUTTON_VALUE_MAX_CHARS: usize = 2000; + #[derive(Deserialize, Debug)] pub struct SlackFormData { payload: String, @@ -127,6 +133,9 @@ struct PrivateMetadata { // HMAC over (w_id, resource_path) keyed on the workspace key; minted when the modal is // built, required by `handle_submission` before the resource_path is decrypted. signature: Option, + // Only selects the wording of the updated channel message, so it is left unsigned. + #[serde(default)] + skin: ApprovalSkin, } // Opportunistic transport-level check: when `SLACK_SIGNING_SECRET` is configured we verify @@ -432,6 +441,7 @@ async fn handle_submission( let container: Container = private_metadata.container; let hide_cancel = private_metadata.hide_cancel; let signature = private_metadata.signature; + let skin = private_metadata.skin; // If hide_cancel is true, we don't need to extract information from the private_metadata if hide_cancel.unwrap_or(false) && action == "cancel" { @@ -463,7 +473,7 @@ async fn handle_submission( tracing::warn!("Failed to resolve slack token for {w_id}/{resource_path}: {e:#}"); Error::BadRequest("Invalid Slack callback request".to_string()) })?; - update_original_slack_message(action, slack_token, container).await?; + update_original_slack_message(action, slack_token, container, skin).await?; Ok(()) } @@ -475,14 +485,19 @@ async fn transform_schemas( required: Option>, default_args_json: Option<&serde_json::Value>, dynamic_enums_json: Option<&serde_json::Value>, + skin: ApprovalSkin, ) -> Result { tracing::debug!("Resume urls: {:#?}", urls); + let link_label = match skin { + ApprovalSkin::Detailed => "Flow suspension details", + ApprovalSkin::Minimal => "View in Windmill", + }; let mut blocks = vec![serde_json::json!({ "type": "section", "text": { "type": "mrkdwn", - "text": format!("{}\n<{}|Flow suspension details>", text, urls.approvalPage), + "text": format!("{}\n<{}|{link_label}>", text, urls.approvalPage), } })]; @@ -918,10 +933,6 @@ async fn send_slack_message( value["approver"] = serde_json::json!(approver); } - if let Some(message) = message { - value["message"] = serde_json::json!(message); - } - if let Some(default_args_json) = default_args_json { value["default_args_json"] = default_args_json.clone(); } @@ -950,33 +961,8 @@ async fn send_slack_message( .map_err(|e| Box::new(e) as Box)?; value["signature"] = serde_json::json!(signature); - let payload = serde_json::json!({ - "channel": channel_id, - "text": "A flow has been suspended. Please approve or reject the flow.", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A flow has been suspended. Please approve or reject the flow." - } - }, - { - "type": "actions", - "elements": [ - { - "type": "button", - "text": { - "type": "plain_text", - "text": "View" - }, - "action_id": "open_modal", - "value": value.to_string() - } - ] - } - ] - }); + let skin = get_approval_step_skin(db, w_id, job_id, flow_step_id).await; + let payload = channel_message_payload(channel_id, skin, message, value); tracing::debug!("Payload: {:?}", payload); @@ -1000,6 +986,88 @@ async fn send_slack_message( Ok(StatusCode::OK) } +/// The channel post announcing the approval. Its button hands `button_value` to the modal, with +/// `message` added, shortened to what Slack's button value limit leaves room for. +fn channel_message_payload( + channel_id: &str, + skin: ApprovalSkin, + message: Option<&str>, + mut button_value: serde_json::Value, +) -> serde_json::Value { + let message = message.map(|m| message_fitting_button_value(&button_value, m)); + if let Some(message) = &message { + button_value["message"] = serde_json::json!(message); + } + let (text, section, button_label) = match skin { + ApprovalSkin::Detailed => { + let text = "A flow has been suspended. Please approve or reject the flow."; + (text, text.to_string(), "View") + } + ApprovalSkin::Minimal => { + let mut section = "*Approval requested*".to_string(); + if let Some(message) = &message { + section.push('\n'); + section.push_str(message); + } + ("Approval requested", section, "Review") + } + }; + + serde_json::json!({ + "channel": channel_id, + "text": text, + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": section + } + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": button_label + }, + "action_id": "open_modal", + "value": button_value.to_string() + } + ] + } + ] + }) +} + +/// The longest prefix of `message` that keeps `button_value` carrying it within Slack's limit. +fn message_fitting_button_value(button_value: &serde_json::Value, message: &str) -> String { + let mut with_message = button_value.clone(); + let mut fits = |max_chars: usize| { + let fitted = truncate_with_ellipsis(message, max_chars); + with_message["message"] = serde_json::json!(fitted); + (with_message.to_string().chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS).then_some(fitted) + }; + if let Some(whole) = fits(usize::MAX) { + return whole; + } + // Searched on the serialized length, which escaping makes longer than the raw prefix, and + // which grows with every character kept. + let (mut shortest, mut longest) = + (0, message.chars().count().min(SLACK_BUTTON_VALUE_MAX_CHARS)); + while shortest < longest { + let mid = (shortest + longest + 1) / 2; + if fits(mid).is_some() { + shortest = mid; + } else { + longest = mid - 1; + } + } + fits(shortest).unwrap_or_else(|| truncate_with_ellipsis(message, 0)) +} + async fn get_modal_blocks( db: DB, w_id: &str, @@ -1034,7 +1102,7 @@ async fn get_modal_blocks( ) .await?; - let ApprovalFormDetails { message_str, urls, schema } = approval_details; + let ApprovalFormDetails { message_str, urls, schema, skin } = approval_details; // Get the card content let card_content = transform_schemas( @@ -1063,6 +1131,7 @@ async fn get_modal_blocks( }), default_args_json, dynamic_enums_json, + skin, ) .await?; @@ -1077,6 +1146,7 @@ async fn get_modal_blocks( resume_button_text, cancel_button_text, &private_metadata_signature, + skin, ))) } @@ -1090,27 +1160,32 @@ fn construct_payload( resume_button_text: Option<&str>, cancel_button_text: Option<&str>, signature: &str, + skin: ApprovalSkin, ) -> serde_json::Value { + let (title, resume_label, cancel_label) = match skin { + ApprovalSkin::Detailed => ("Workflow Suspended", "Resume Workflow", "Cancel Workflow"), + ApprovalSkin::Minimal => ("Approval request", "Approve", "Reject"), + }; let mut view = serde_json::json!({ "type": "modal", "callback_id": "submit_form", "notify_on_close": true, "title": { "type": "plain_text", - "text": "Workflow Suspended" + "text": title }, "blocks": blocks, "submit": { "type": "plain_text", - "text": resume_button_text.unwrap_or("Resume Workflow") + "text": resume_button_text.unwrap_or(resume_label) }, - "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature }).to_string(), + "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature, "skin": skin }).to_string(), }); if !hide_cancel { view["close"] = serde_json::json!({ "type": "plain_text", - "text": cancel_button_text.unwrap_or("Cancel Workflow") + "text": cancel_button_text.unwrap_or(cancel_label) }); } @@ -1193,11 +1268,13 @@ async fn update_original_slack_message( action: &str, token: String, container: Container, + skin: ApprovalSkin, ) -> Result<(), Error> { - let message = if action == "resume" { - "\n\n*Workflow has been resumed!* :white_check_mark:" - } else { - "\n\n*Workflow has been canceled!* :x:" + let message = match (skin, action == "resume") { + (ApprovalSkin::Detailed, true) => "\n\n*Workflow has been resumed!* :white_check_mark:", + (ApprovalSkin::Detailed, false) => "\n\n*Workflow has been canceled!* :x:", + (ApprovalSkin::Minimal, true) => "*Approved* :white_check_mark:", + (ApprovalSkin::Minimal, false) => "*Rejected* :x:", }; let final_blocks = vec![serde_json::json!({ @@ -1242,3 +1319,66 @@ async fn update_original_slack_message( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_message_keeps_the_channel_post_within_slack_limits() { + let button_value = serde_json::json!({ + "w_id": "demo", + "job_id": Uuid::nil(), + "path": "u/admin/slack", + "channel": "C0123456789", + "flow_step_id": "a", + "signature": "f".repeat(64), + }); + let carried = |skin, message: &str| { + let payload = channel_message_payload("C1", skin, Some(message), button_value.clone()); + let button = payload["blocks"][1]["elements"][0]["value"] + .as_str() + .unwrap() + .to_string(); + assert!(button.chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS); + let section = payload["blocks"][0]["text"]["text"].as_str().unwrap(); + assert!(section.chars().count() <= 3000); + serde_json::from_str::(&button) + .unwrap() + .message + .unwrap() + }; + // Quotes and newlines each cost two characters once escaped into the button value. + let message = "Expense \"offsite\" line\n".repeat(1000); + for skin in [ApprovalSkin::Detailed, ApprovalSkin::Minimal] { + let kept = carried(skin, &message); + let kept = kept.strip_suffix("...").unwrap(); + assert!(message.starts_with(kept)); + assert!(kept.chars().count() > 1_000); + assert_eq!(carried(skin, "Short message"), "Short message"); + } + } + + #[test] + fn minimal_skin_survives_the_modal_round_trip() { + let container = Container { message_ts: "1".to_string(), channel_id: "C1".to_string() }; + let payload = construct_payload( + serde_json::json!([]), + false, + "trigger", + "https://example.com/resume", + "u/admin/slack", + container, + None, + None, + "signature", + ApprovalSkin::Minimal, + ); + let view = &payload["view"]; + assert_eq!(view["submit"]["text"], "Approve"); + assert_eq!(view["close"]["text"], "Reject"); + let metadata: PrivateMetadata = + serde_json::from_str(view["private_metadata"].as_str().unwrap()).unwrap(); + assert_eq!(metadata.skin, ApprovalSkin::Minimal); + } +} diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 06a79d022e..22645a2ac4 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -107,6 +107,14 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< .header("Cross-Origin-Resource-Policy", "cross-origin"); } + // Login and its siblings carry a different `rd` on every page that links + // to them, so a crawler meets thousands of URLs for one form. The app is + // client-rendered, so a meta tag only exists after a render pass; the + // header is seen on the first fetch. + if original_path.starts_with("/user/") { + res = res.header("X-Robots-Tag", "noindex, nofollow"); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index e58aab44a2..387ba56589 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -1,6 +1,7 @@ use super::{ - http_trigger_args::RawHttpTriggerArgs, refresh_routers, AuthenticationMethod, HttpMethod, - RequestType, TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, + effective_allowed_origins, http_trigger_args::RawHttpTriggerArgs, match_origin, + refresh_routers, AuthenticationMethod, HttpMethod, RequestType, TriggerRoute, + HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, }; use crate::{ auth::{AuthCache, OptTokened}, @@ -24,6 +25,7 @@ use std::{collections::HashMap, sync::Arc}; use windmill_common::{ db::UserDB, error::{Error, Result}, + global_settings::HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS, jobs::JobTriggerKind, triggers::{TriggerKind, TriggerMetadata}, utils::{not_found_if_none, StripPath}, @@ -37,12 +39,222 @@ use { windmill_object_store::build_object_store_client, }; +/// Which router a request's CORS decision must be looked up in. +/// +/// A preflight names the method it is asking about in +/// `Access-Control-Request-Method`; the routers are keyed by method, so without +/// that header there is nothing to look up. +fn cors_lookup_method(req: &axum::extract::Request) -> Option { + let method = req.method(); + if method == http::Method::OPTIONS { + req.headers() + .get(http::header::ACCESS_CONTROL_REQUEST_METHOD) + .and_then(|method| method.to_str().ok()) + .and_then(|method| http::Method::try_from(method).ok()) + .as_ref() + .and_then(routable_method) + } else { + routable_method(method) + } +} + +/// The router key a request method maps to. `HEAD` resolves the `GET` route it +/// mirrors, and does so for a preflight naming it too: browsers send +/// `Access-Control-Request-Method: HEAD` when the HEAD carries a non-safelisted +/// header, and answering that preflight from a different route than the request +/// itself resolves is how the two come to disagree. +fn routable_method(method: &http::Method) -> Option { + if method == http::Method::HEAD { + Some(HttpMethod::Get) + } else { + HttpMethod::try_from(method).ok() + } +} + +/// The key to look a request up by, matching what `route_job` resolves it to. +/// +/// `Path` percent-decodes before `get_http_route_trigger` builds its +/// lookup key, so decoding here is what keeps the two agreeing: on the raw path, +/// `/us%65rs` misses the trigger registered at `/users` that goes on to serve the +/// request, and the response would carry the permissive default instead of that +/// trigger's allowlist. +fn cors_lookup_path(raw_path: &str) -> Option { + let decoded = urlencoding::decode(raw_path).ok()?; + // `StripPath::to_path` strips one leading slash and the handler trims + // trailing ones, before a single `/` is prefixed back on. + let stripped = decoded.strip_prefix('/').unwrap_or(&decoded); + Some(format!("/{}", stripped.trim_end_matches('/'))) +} + +/// What the middleware should stamp, decided while the routers guard is held. +/// +/// Deliberately small and owned: the allowlist itself never leaves the guard, +/// so a large one is scanned in place instead of being copied per request onto +/// a path an unauthenticated preflight can reach. +#[derive(Clone)] +enum CorsDecision { + /// No allowlist applies, so the permissive default stands. + Unrestricted, + /// An allowlist applies. `allow_origin` is the value to echo, present only + /// when the request's own `Origin` is on the list. + Restricted { route_method: Option, allow_origin: Option }, + /// The routers could not be read, so nothing is known about this path. + Unavailable, +} + +/// Whether a route actually serves a static website, rather than merely saying +/// it does. +/// +/// `is_static_website` is a caller-set flag that validation ties to nothing: a +/// route can carry it while having no assets configured and a `script_path` +/// that `route_job` runs regardless. Keying the exemption off the flag alone +/// would let one boolean disable a route's allowlist and hand its runnable back +/// the `wm_headers` escape hatch, so the assets have to be there too. +fn serves_a_static_website(trigger: &TriggerRoute) -> bool { + trigger.is_static_website && trigger.static_asset_config.is_some() +} + +/// A static website is never subject to an allowlist, its own included. It has +/// no authentication of its own — the editor does not offer any — so it hands +/// out public files that any non-browser client can already fetch, and +/// restricting which browsers may read them protects nothing while breaking the +/// cross-origin uses that do consult CORS: a webfont, a `crossorigin` asset, a +/// `fetch`. +/// +/// A single-file static asset is not exempt. That one can carry an +/// `authentication_method`, so its content need not be public, and an allowlist +/// is what keeps another origin from reading a response its own credentials +/// would not have obtained. +/// +/// The CORS verdict for a request, published by whoever resolved its trigger. +/// +/// The middleware stamps headers after the handler returns, but only the +/// handler knows which trigger it actually served. Re-deriving that from the +/// routers cache is a second lookup which can disagree with the first when a +/// route is edited, deleted or widened mid-request, and every ordering of the +/// two is wrong in some case. So the verdict travels with the request instead +/// of being worked out twice. +#[derive(Clone, Default)] +struct ResolvedCorsPolicy(std::sync::Arc>); + +impl ResolvedCorsPolicy { + /// Record what the trigger being served allows. Called once, where the + /// route is resolved, so the answer cannot drift from the response. + fn publish(&self, trigger: &TriggerRoute, method: Option, headers: &HeaderMap) { + let decision = if serves_a_static_website(trigger) { + CorsDecision::Unrestricted + } else { + let instance_default = HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.load(); + match effective_allowed_origins( + trigger.allowed_origins.as_deref(), + instance_default.as_slice(), + ) { + None => CorsDecision::Unrestricted, + Some(allowed_origins) => CorsDecision::Restricted { + route_method: method, + allow_origin: match_origin(allowed_origins, headers.get(http::header::ORIGIN)), + }, + } + }; + let _ = self.0.set(decision); + } + + fn published(&self) -> Option { + self.0.get().cloned() + } +} + +/// Decide the CORS answer from the routers cache, for a request no handler +/// published a verdict for: a preflight, an unknown path, or any failure ahead +/// of the publish — authentication included, which runs after the route itself +/// resolves. +/// +/// Loads the routers when the cache is cold, the way `get_http_route_trigger` +/// does, so a preflight is answered from the same view of the routes as the +/// request that follows it. +async fn resolve_cors_decision( + db: &DB, + http_method: HttpMethod, + requested_path: &str, + origin: Option<&http::HeaderValue>, +) -> CorsDecision { + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + + let routers_cache = if routers_cache.routers.is_empty() { + drop(routers_cache); + match refresh_routers(db, false).await { + Ok((_, routers_cache)) => routers_cache, + Err(err) => { + tracing::error!("Could not load HTTP routers to resolve CORS: {err:#}"); + return CorsDecision::Unavailable; + } + } + } else { + routers_cache + }; + + let Some(router) = routers_cache.routers.get(&http_method) else { + return CorsDecision::Unavailable; + }; + + let route = router.at(requested_path).ok(); + if route + .as_ref() + .is_some_and(|trigger| serves_a_static_website(trigger.value)) + { + return CorsDecision::Unrestricted; + } + let route_allowed_origins = route + .as_ref() + .and_then(|trigger| trigger.value.allowed_origins.as_deref()); + + let instance_default = HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.load(); + match effective_allowed_origins(route_allowed_origins, instance_default.as_slice()) { + None => CorsDecision::Unrestricted, + Some(allowed_origins) => CorsDecision::Restricted { + route_method: route.map(|_| http_method), + allow_origin: match_origin(allowed_origins, origin), + }, + } +} + async fn conditional_cors_middleware( - req: axum::extract::Request, + Extension(db): Extension, + mut req: axum::extract::Request, next: axum::middleware::Next, ) -> Response { + let origin = req.headers().get(http::header::ORIGIN).cloned(); + // Owned before `next.run` consumes the request. `&Request` is not `Send` + // (`Body` is not `Sync`), so nothing borrowed from it can cross the await. + // The URI is carried rather than the decoded path: cloning it is a refcount + // bump, while decoding allocates, and only the fallback below ever needs it. + let lookup_method = cors_lookup_method(&req); + let uri = req.uri().clone(); + + let resolved = ResolvedCorsPolicy::default(); + req.extensions_mut().insert(resolved.clone()); + let mut response = next.run(req).await; + let decision = match resolved.published() { + // The handler resolved a trigger and said what it served under. That is + // the policy this response was produced with, so nothing else can be + // more authoritative. + Some(decision) => decision, + // No verdict was published: a preflight, an unknown path, or a request + // that failed before reaching the publish, authentication included. No + // runnable produced this body, so reading the cache cannot contradict + // anything. + None => match lookup_method.zip(cors_lookup_path(uri.path())) { + Some((method, path)) => { + resolve_cors_decision(&db, method, &path, origin.as_ref()).await + } + // Not a preflight, not a routable method, or a path that does not + // decode. + None => CorsDecision::Unrestricted, + }, + }; + let headers = response.headers_mut(); // Check existing headers first to determine what not to insert @@ -67,18 +279,65 @@ async fn conditional_cors_middleware( } } - // Insert only the missing headers - if !not_insert_origin { - headers.insert( - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, - http::HeaderValue::from_static("*"), - ); + match &decision { + CorsDecision::Restricted { allow_origin, .. } => { + // A configured allowlist decides, overriding any `wm_headers` value + // the runnable set. The preflight is answered before any code runs, + // so config is the only thing it can consult; letting the response + // widen what the preflight advertised would make the two disagree + // and leave the allowlist bounding nothing. A route escapes a + // stricter instance default — `wm_headers` included — by setting + // its own list to `*`. + match allow_origin { + Some(value) => { + headers.insert(http::header::ACCESS_CONTROL_ALLOW_ORIGIN, value.clone()) + } + // No match: omit the header entirely so the browser blocks the + // read, and drop any value the runnable set. + None => headers.remove(http::header::ACCESS_CONTROL_ALLOW_ORIGIN), + }; + // Appended, not inserted: the answer now depends on the request's + // Origin, and a shared cache that ignores it would hand one + // origin's response to another. + headers.append(http::header::VARY, http::HeaderValue::from_static("origin")); + } + // The routers could not be read, so nothing is known about this path; + // only a preflight or an unresolved request reaches here. Answering a + // preflight permissively would let a disallowed origin go on to invoke + // a runnable whose purpose may be a side effect. + CorsDecision::Unavailable => { + headers.remove(http::header::ACCESS_CONTROL_ALLOW_ORIGIN); + } + CorsDecision::Unrestricted => { + if !not_insert_origin { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, + http::HeaderValue::from_static("*"), + ); + } + } } if !not_insert_methods { + // A route accepts exactly one method, so advertising all seven + // overstates it. Only a route under an allowlist gets the narrower + // answer; an unrestricted one advertises the full supported set, since + // narrowing it would say something about a route the response is not + // otherwise willing to disclose. + let restricted_method = match &decision { + CorsDecision::Restricted { route_method, .. } => *route_method, + _ => None, + }; headers.insert( http::header::ACCESS_CONTROL_ALLOW_METHODS, - http::HeaderValue::from_static("GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"), + http::HeaderValue::from_static(match restricted_method { + Some(HttpMethod::Get) => "GET, OPTIONS", + Some(HttpMethod::Post) => "POST, OPTIONS", + Some(HttpMethod::Put) => "PUT, OPTIONS", + Some(HttpMethod::Delete) => "DELETE, OPTIONS", + Some(HttpMethod::Patch) => "PATCH, OPTIONS", + None => "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS", + }), ); } @@ -237,6 +496,7 @@ async fn route_job( Extension(db): Extension, Extension(user_db): Extension, Extension(auth_cache): Extension>, + Extension(cors_policy): Extension, OptTokened { token }: OptTokened, Path(route_path): Path, headers: HeaderMap, @@ -255,6 +515,10 @@ async fn route_job( .await .map_err(|e| e.into_response())?; + // Publish before anything else can fail: the CORS middleware stamps this + // response either way, and it must reflect the trigger actually served. + cors_policy.publish(&trigger, routable_method(&args.0.metadata.method), &headers); + if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { return Err(Error::NotFound(format!( "Runnable path of HTTP route at path: {}", diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index ed66eb5e74..4dd073b3af 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -261,6 +261,12 @@ async fn rename_user( ))); } + let old_instance_username = + sqlx::query_scalar!("SELECT username FROM password WHERE email = $1", user_email) + .fetch_optional(&mut *tx) + .await? + .flatten(); + sqlx::query!( "UPDATE password SET username = $1 WHERE email = $2", ru.new_username, @@ -269,6 +275,36 @@ async fn rename_user( .execute(&mut *tx) .await?; + // The per-workspace sweep below only reaches accounts with a `usr` row. A superadmin acting + // outside their workspaces has none, yet an app can name them: their principal is + // `u/{password.username}`, which this rename just moved. Matching on the address as well + // keeps a like-named member of some other workspace out of it. + if let Some(old_username) = old_instance_username.filter(|u| *u != ru.new_username) { + let old_principal = windmill_common::users::username_to_permissioned_as(&old_username); + let new_principal = + windmill_common::users::username_to_permissioned_as(&ru.new_username); + sqlx::query!( + "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)) + WHERE policy->>'on_behalf_of' = $2 AND policy->>'on_behalf_of_email' = $3", + &new_principal, + &old_principal, + user_email + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' = $2 + AND value->'policy'->>'on_behalf_of_email' = $3"#, + &new_principal, + &old_principal, + user_email + ) + .execute(&mut *tx) + .await?; + } + let workspace_usernames = sqlx::query!( "SELECT workspace_id, username FROM usr WHERE email = $1", &user_email @@ -754,6 +790,17 @@ async fn update_username_in_workpsace<'c>( .execute(&mut **tx) .await?; + // An app draft carries a copy of the deployed policy, so the rename must reach it + // there too — same reason as the script/flow draft sweep above. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb('u/' || $1))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( "UPDATE app SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", new_username, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index da8532e9d1..a0a27eab4c 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -108,6 +108,7 @@ async fn edit_copilot_config( } ai_config.validate_model_pricing()?; + ai_config.validate_sessions_retention()?; let mut tx = db.begin().await?; @@ -147,6 +148,8 @@ async fn edit_copilot_config( let workspace_has_config = ai_config.has_providers(); let copilot_disabled = ai_config.copilot_disabled; + let sessions_storage_disabled = ai_config.sessions_storage_disabled; + let sessions_retention_days = ai_config.sessions_retention_days; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -174,6 +177,8 @@ async fn edit_copilot_config( AIConfig::default() }; effective_ai_config.copilot_disabled = copilot_disabled; + effective_ai_config.sessions_storage_disabled = sessions_storage_disabled; + effective_ai_config.sessions_retention_days = sessions_retention_days; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -212,6 +217,12 @@ async fn get_copilot_info( let copilot_disabled = workspace_ai_config .as_ref() .is_some_and(|c| c.0.copilot_disabled); + let sessions_storage_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.sessions_storage_disabled); + let sessions_retention_days = workspace_ai_config + .as_ref() + .and_then(|c| c.0.sessions_retention_days); let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -236,6 +247,8 @@ async fn get_copilot_info( AIConfig::default() }; effective.copilot_disabled = copilot_disabled; + effective.sessions_storage_disabled = sessions_storage_disabled; + effective.sessions_retention_days = sessions_retention_days; Ok(Json(effective)) } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 05cd354cd4..69565ffcbd 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -149,9 +149,7 @@ async fn derive_email( if let Some(hit) = cache.get(permissioned_as) { return Ok(Some(hit.clone())); } - // Uncached: the address goes into an archive a client redeploys from, and the write path - // validates the pair it sends back against an uncached lookup. The memo above still holds - // this to one query per distinct principal per export. + // The memo above holds this to one query per distinct principal per export. let email = windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) .await?; @@ -346,7 +344,7 @@ pub(crate) struct ArchiveQueryParams { default_ts: Option, /// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format settings_version: Option, - /// Opt-in: include `extra_perms` on flow / script / app rows. Default `false` + /// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false` /// so cross-workspace tarball imports do not carry over ACLs referring to /// identities that may not exist in the target workspace. `wmill sync pull` /// passes `true` to surface ACLs in the git-tracked yaml. @@ -365,8 +363,8 @@ pub(crate) struct ArchiveQueryParams { /// pre-existing serialization for folders and groups so /// no customer sees a one-time noisy diff on upgrade. /// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}` -/// or null. New surface for flow / script / app, which -/// never carried ACLs in source before this change. +/// or null. New surface for script / flow / app / variable, +/// which never carried ACLs in source before this change. #[derive(Clone, Copy)] pub enum ExtraPermsBehavior { Drop, @@ -665,7 +663,7 @@ pub(crate) async fn tarball_workspace( check_scopes(&authed, || "variables:read".to_string())?; } - // Opt-in behavior for surfacing per-resource ACLs on flow/app rows. + // Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows. // Folder and group rows have always carried `extra_perms` in source and // continue to do so unconditionally (`KeepEvenEmpty`) so existing // customer git repos see no one-time noisy diff. @@ -926,7 +924,7 @@ pub(crate) async fn tarball_workspace( if !skip_resource_types.unwrap_or(false) { let resource_types = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", &w_id ) .fetch_all(&mut *tx) @@ -1002,8 +1000,7 @@ pub(crate) async fn tarball_workspace( Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) })?); } - let var_str = - &to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap(); + let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) .await?; @@ -1435,9 +1432,11 @@ pub(crate) async fn tarball_workspace( // Native triggers (Nextcloud, Google Drive, GitHub) are never // cloned into a fork — a fork only has one if its owner created // it there, so it's always "fork-only" and keeps its own mode. - // No parent-value substitution applies; we only strip the - // webhook token hash. - let native_ignore_keys = vec!["webhook_token_hash"]; + // No parent-value substitution applies; we strip the webhook + // token hash, and `enabled`, which is operational state a sync + // deliberately does not carry — whether a trigger is paused + // belongs to the workspace it runs in, not to the code. + let native_ignore_keys = vec!["webhook_token_hash", "enabled"]; for trigger in native_triggers { let trigger_str = &to_string_without_metadata( @@ -1587,10 +1586,11 @@ pub(crate) async fn tarball_workspace( .await?; // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility - // Server-owned auto-pull state (the HMAC webhook secret + hook id/error and - // the synced-sha / last-pull status) must never leave the server: keep it out - // of export archives and synced repos, and don't let a re-imported workspace - // inherit another install's hook/sync state. Mirrors the GET-settings redaction. + // Server-owned state (the HMAC webhook secret + hook id/error, the + // synced-sha / last-pull status, the admin automatic pulls run as, and what + // the credential check observed) must never leave the server: keep it out of + // export archives and synced repos, and don't let a re-imported workspace + // inherit another install's hook/sync state or pull identity. fn redact_git_sync_for_export(git_sync: Option) -> Option { let mut git_sync = git_sync?; if let Some(repos) = git_sync @@ -1607,10 +1607,18 @@ pub(crate) async fn tarball_workspace( "webhook_error", "last_synced_sha", "last_pull_status", + "enabled_by", ] { auto_pull.remove(field); } } + // What this install observed about its own credential: a token + // id and expiry, and a `checked_at` that moves on its own. + // None of it describes the workspace, and in a git-synced + // `wmill.yaml` it would churn the file for no reason. + if let Some(repo) = repo.as_object_mut() { + repo.remove("credential"); + } } } Some(git_sync) diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 117b41632a..4d5523524c 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -175,10 +175,12 @@ fn is_write_access(access: Option) -> bool { /// producers). Resource / datatable / volume reads stay explicit-`// on`: /// a config/lookup read cascading is more often surprising than wanted. fn is_auto_trigger_kind(kind: AssetKind) -> bool { - // `Dbt` is deliberately NOT here. dbt is the only thing that can produce a - // warehouse relation (`// materialize` takes DuckLake targets only) and a dbt - // run does not dispatch, so a derived `dbt://` edge could never fire — it - // would draw a cascade arrow into a script nothing can wake. + // `Dbt` is deliberately NOT here. A warehouse relation is usually built by + // the dbt project that reads it, and a dbt run does not dispatch, so deriving + // an edge from every `dbt://` read would draw cascade arrows that mostly never + // fire. The relations a native `// materialize manual dbt://…` script writes + // do wake subscribers, but only through an explicit `// on`, which is where + // the author states that this particular relation has such a producer. matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) } @@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs( out } +/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is +/// its ONLY producer. +/// +/// That is the one shape in which subscribing to a warehouse relation can never +/// be woken: a dbt run records the models it built and does not dispatch +/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that +/// declares `// materialize manual dbt://…` fans out on the ordinary path. +/// +/// `None` covers both "some non-dbt script materializes it" and "nothing +/// produces it yet" — the second is the ordinary deploy-order case, identical to +/// every other asset kind, not a dormant edge. +/// +/// **Give it the workspace pool, not an RLS-scoped transaction.** `script` +/// carries RLS while `asset` does not, so a scoped executor hides producers, and +/// the hidden ones fail in the harmful direction: a native producer the deployer +/// cannot read leaves a dbt-only set behind and refuses a subscription that would +/// have fired. What it discloses in exchange is the path of a dbt script building +/// a relation the caller already named, which the workspace asset graph hands out +/// for every `dbt://` node anyway (the source that script wrote stays gated). +/// Callers must therefore already be scoped to `workspace_id`. +/// +/// `deploying_paths` is excluded from the producer set, and has to be: reading +/// committed rows means the deploying script's own are the version being +/// replaced, so one that just dropped its `// materialize` would still count as a +/// producer and let a now-dormant subscription through. Pass every path this +/// deploy is rewriting — under a rename that is the old path as well as the new +/// one, whose committed write row the transaction is about to remove. Excluding +/// them is free of the opposite error, because a script never wakes its own +/// subscription — the dispatcher skips that as a self-loop. +/// +/// A producer another deploy is committing concurrently is invisible either way, +/// and the outcome depends on which side it is. An uncommitted NATIVE producer +/// leaves a dbt-only set and refuses, with a message the user can retry past. An +/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then +/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger +/// row lands, neither side reports the edge it left dormant. Serializing the two +/// is not worth it: they would have to share a per-relation lock, and the ingest +/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding +/// relation locks first inverts that order into a deadlock across the two +/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md). +pub async fn sole_dbt_producer<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_path: &str, + deploying_paths: &[String], +) -> error::Result> { + use crate::scripts::ScriptLang; + let producers = sqlx::query!( + r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang" + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2 + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND a.usage_path <> ALL($3)"#, + workspace_id, + asset_path, + deploying_paths + ) + .fetch_all(executor) + .await?; + if producers + .iter() + .any(|p| !matches!(p.language, ScriptLang::Dbt)) + { + return Ok(None); + } + Ok(producers.into_iter().next().map(|p| p.path)) +} + +/// The set form of [`sole_dbt_producer`], for asking about many relations at +/// once: every `// on dbt://` edge among `relations` whose producers +/// are all dbt scripts, rendered as `dbt:// → `. +/// +/// A dbt deploy asks this about the relations it just ingested, because that +/// ingest is what can retroactively leave a subscription accepted earlier — when +/// nothing produced the relation — with dbt as its only producer. +/// +/// Spells the predicate the same way its singular sibling does, per subscriber: +/// the producer set excludes the subscriber's own path (a script never wakes +/// itself) and has to be non-empty (nothing produces it yet is deploy order, not +/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would +/// silence this warning with nothing failing. +/// +/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace +/// pool, caller already scoped to `workspace_id`. +pub async fn dormant_dbt_subscriptions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + relations: &[String], +) -> error::Result> { + if relations.is_empty() { + return Ok(vec![]); + } + let refs = relations + .iter() + .map(|r| format!("dbt://{r}")) + .collect::>(); + Ok(sqlx::query_scalar!( + r#"WITH producer AS ( + SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND 'dbt://' || a.path = ANY($2) + ) + SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!" + FROM script_trigger st + WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset' + AND st.trigger_ref = ANY($2) + AND EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language = 'dbt') + AND NOT EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language <> 'dbt') + ORDER BY 1"#, + workspace_id, + &refs + ) + .fetch_all(executor) + .await?) +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index b51186f464..73cdfba35d 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -452,6 +452,42 @@ async fn fetch_authed_from_permissioned_as_inner( w_id: &str, conn: &mut sqlx::PgConnection, ) -> Result { + // The `usr` row is the live binding between a `u/` principal and an address, and it is read + // here anyway for the workspace role. Callers may hand us a cached address, so read it before + // anything is granted: `super_admin` and `email_to_igroup` below are keyed on the address + // while the role is keyed on the principal, and an address that no longer belongs to this + // principal — a username freed and reassigned while its previous holder keeps a privileged + // account — would mix one account's role with another's instance privileges. + let member = match permissioned_as.split_once('/') { + Some(("u", name)) => sqlx::query!( + "SELECT is_admin, operator, email FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id + ) + .fetch_optional(&mut *conn) + .await?, + _ => None, + }; + let resolved_email; + let email = match member.as_ref() { + Some(m) => m.email.as_str(), + // No enabled `usr` row. Resolve as `resolve_username_to_email` does: a disabled member's + // own row still wins over the `password` superadmin fallback, so it can never resolve to + // an unrelated superadmin who shares the username (workspace usernames are only unique per + // workspace). Off the member path, which is why it is worth a query that path skips. + None => match permissioned_as.split_once('/') { + Some(("u", name)) => { + resolved_email = + crate::users::resolve_username_to_email(w_id, name, &mut *conn).await?; + // No live binding at all: the supplied address stands. A cached one is at most one + // notify poll stale; accepted, see `users::get_email_from_permissioned_as`. + resolved_email.as_deref().unwrap_or(email) + } + _ => email, + }, + }; + let is_super_admin = permissioned_as == SUPERADMIN_SYNC_EMAIL || email == SUPERADMIN_SECRET_EMAIL || email == SUPERADMIN_NOTIFICATION_EMAIL @@ -465,22 +501,12 @@ async fn fetch_authed_from_permissioned_as_inner( if prefix == "u" { let (is_admin, is_operator) = if is_super_admin { (true, false) + } else if let Some(m) = member.as_ref() { + (m.is_admin, m.operator) } else { - let r = sqlx::query!( - "SELECT is_admin, operator FROM usr where username = $1 AND \ - workspace_id = $2 AND disabled = false", - name, - &w_id - ) - .fetch_optional(&mut *conn) - .await?; - if let Some(r) = r { - (r.is_admin, r.operator) - } else { - return Err(Error::NotFound(format!( - "user {name} not found in workspace {w_id}" - ))); - } + return Err(Error::NotFound(format!( + "user {name} not found in workspace {w_id}" + ))); }; let groups = get_groups_for_user(w_id, &name, email, &mut *conn).await?; diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index 8fb124a33f..157a61f129 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -8,9 +8,11 @@ //! Two things this module is deliberate about: //! //! * **Asset identity is the physical relation.** A model becomes -//! `dbt:////`: the scheme names dbt, which is the -//! only thing that creates one, but the PATH is the relation and never dbt's -//! own `unique_id`. Two projects meet at a handoff — one materializes a mart, +//! `dbt:////`: the scheme names the namespace dbt +//! made — it is the only thing that DERIVES one, while any other language can +//! declare a write to one (decision 25) — but the PATH is the relation and +//! never dbt's own `unique_id`. Two projects meet at a handoff — one +//! materializes a mart, //! the next declares it a `source` — where `model.a.orders` and //! `source.b.analytics.orders` differ but the relation does not, so keying on //! the node id would leave each project an island; a native script reading the @@ -33,8 +35,8 @@ //! Every `pub` mutator in this module — the manifest ones //! (`replace_dbt_manifest`, `clear_dbt_manifest_version`, //! `clear_dbt_editor_graphs`), -//! the snapshot sweep, and the retry-state ones (`move_dbt_run_state`, -//! `clear_dbt_run_state`, `clear_dbt_run_state_if_path_retired`) — takes the +//! the snapshot sweep, and the script-state ones (`move_dbt_script_state`, +//! `clear_dbt_script_state`, `clear_dbt_script_state_if_path_retired`) — takes the //! workspace and the script to act on as plain arguments and enforces nothing: //! **the caller must already have verified write access to that script**, //! exactly like the sibling `assets::replace_static_asset_usage` each is called @@ -188,6 +190,19 @@ fn graph_digest(ingested: &IngestedManifest, relation_root: &str) -> String { .unwrap_or_default() .as_bytes(), ); + // Only when there are any, so a project that never asked for the analysis + // pass keeps the digest it already has. Hashing an empty section + // unconditionally would change every stored digest at once, and every + // dynamic run would then store a full snapshot until its script is + // redeployed — which reads exactly like the suppression above never working. + if !ingested.column_edges.is_empty() { + h.update(b"\0"); + h.update( + serde_json::to_string(&ingested.column_edges) + .unwrap_or_default() + .as_bytes(), + ); + } format!("{:x}", h.finalize()) } @@ -297,6 +312,14 @@ pub async fn prune_dbt_run_graphs( ) .execute(db) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE job_id <> '00000000-0000-0000-0000-000000000000' + AND ingested_at < now() - make_interval(days => $1)", + RUN_GRAPH_RETENTION_DAYS, + ) + .execute(db) + .await?; // In ONE transaction with the orphan sweep: a restart in the gap leaves graph // rows whose marker is gone, and since the sweep runs only when a marker went, // every later call computes `retired == 0` and skips them for good. @@ -325,7 +348,7 @@ pub async fn prune_dbt_run_graphs( // partial index here — all of them `WHERE job_id <> DEPLOYED` — and past the // keep-count is rare, so the ordinary run should pay for neither. if retired > 0 { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} t WHERE t.workspace_id = $1 AND t.script_path = $2 @@ -381,6 +404,18 @@ pub struct IngestedNode { pub severity: Option, pub attached_node: Option, pub columns: Option, + /// The node's real columns, typed and ordered — `[{"name": …, "type": …}]`, + /// from the engine's static analysis. `None` when the project did not ask + /// for it or the engine wrote none. Beside `columns` rather than merged into + /// it: that one is what the author DECLARED, and stays that. + /// + /// Skipped when absent, unlike its neighbours, because `graph_digest` + /// serializes these nodes: emitting `"column_schema":null` would change + /// every stored digest at once, and every dynamic run of a project that + /// never asked for the pass would store a full snapshot until its script is + /// redeployed. + #[serde(skip_serializing_if = "Option::is_none")] + pub column_schema: Option, pub freshness: Option, /// The transform itself, for the graph to render. The copy taken at /// deploy: the file itself is in the script's module bundle. @@ -388,6 +423,40 @@ pub struct IngestedNode { pub original_file_path: Option, } +/// One column-to-column edge of the ingested graph. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +#[serde(default)] +pub struct IngestedColumnEdge { + pub parent_unique_id: String, + pub parent_column: String, + pub child_unique_id: String, + pub child_column: String, + /// dbt's own word: `copy`, `mod` or `scan`. Kept verbatim — the engine's own + /// reader maps those three and passes anything else through, so the set is + /// open. + pub lineage_kind: String, +} + +/// One column of a node, as the engine's static analysis resolved it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexedColumn { + pub name: String, + /// The declared type where `schema.yml` gives one, else the inferred one. + /// Empty when neither is known; the column still belongs to the relation, so + /// only the type is left out. + pub column_type: String, + /// Position in the relation, which is the order the panel lists them in. + pub index: i64, +} + +/// What one `--write-index` pass produced: the column edges of the whole +/// project and the real column schema per node. +#[derive(Debug, Default)] +pub struct ColumnIndex { + pub edges: Vec, + pub columns: HashMap>, +} + // Serde: an agent worker cannot write these tables directly, so it posts the // whole manifest to the server, which stores it with the same function the SQL // path uses. @@ -398,6 +467,9 @@ pub struct IngestedNode { pub struct IngestedManifest { pub nodes: Vec, pub edges: Vec<(String, String)>, + /// Column-to-column lineage, when the project asked for it and the engine + /// produced it. Empty is the normal case — see `attach_column_index`. + pub column_edges: Vec, /// The `asset` rows the owning script produces (models) and consumes /// (sources) — what the lineage graph is drawn from. pub assets: Vec, @@ -405,6 +477,88 @@ pub struct IngestedManifest { pub adapter_type: String, } +/// The most column edges one graph stores. +/// +/// A `scan` edge — the column was read to produce the row, not the value — is +/// emitted from every join key and every predicate column to every output +/// column, so one wide model over a multi-column join contributes columns times +/// predicates edges on its own. The cap is what keeps a project shaped like that +/// from turning one deploy into a multi-million-row insert; past it the lineage +/// is truncated and the rest of the graph is unaffected. +pub const MAX_COLUMN_EDGES: usize = 200_000; + +/// Whether the value travelled along this edge, as opposed to the column merely +/// being read to produce the row. +/// +/// A `scan` edge reaches every output column of its model, so it is most of what +/// a wide project's index holds and the first thing `MAX_COLUMN_EDGES` gives up. +/// It is still stored, for a view that wants indirect influence. +pub fn is_direct(lineage_kind: &str) -> bool { + matches!(lineage_kind, "copy" | "mod") +} + +impl IngestedManifest { + /// Fold one `--write-index` pass into the graph. + /// + /// Both halves are scoped to the nodes this graph already kept: the index + /// describes the whole project, while the graph describes what this script's + /// selection builds plus the parents anchoring its edges, and an edge whose + /// endpoint is absent has nothing to draw. + pub fn attach_column_index(&mut self, index: ColumnIndex) { + let kept: std::collections::HashSet<&str> = + self.nodes.iter().map(|n| n.unique_id.as_str()).collect(); + let mut edges: Vec = index + .edges + .into_iter() + .filter(|e| { + kept.contains(e.parent_unique_id.as_str()) + && kept.contains(e.child_unique_id.as_str()) + }) + .collect(); + // Sorted and deduplicated for the digest, which decides whether a run + // stores a snapshot at all: parquet row order is the engine's and two + // passes over one project must not read as two different graphs. + // + // Direct kinds first, so what the truncation below gives up is `scan` — + // the bulk of a wide project's lineage, and the kind that says the column + // was read to produce the row rather than the value. The + // worker's reader already applies this order while decoding, because the + // memory bound has to; repeating it here is what makes the ordering a + // property of the manifest rather than of one caller's reader, and it is + // the only ordering an index assembled some other way would get. + edges.sort_by(|a, b| { + is_direct(&b.lineage_kind) + .cmp(&is_direct(&a.lineage_kind)) + .then_with(|| a.cmp(b)) + }); + edges.dedup(); + edges.truncate(MAX_COLUMN_EDGES); + self.column_edges = edges; + + let mut columns = index.columns; + for node in self.nodes.iter_mut() { + let Some(mut cols) = columns.remove(&node.unique_id) else { + continue; + }; + if cols.is_empty() { + continue; + } + cols.sort_by_key(|c| c.index); + node.column_schema = Some(serde_json::Value::Array( + cols.into_iter() + // A column the analysis typed as nothing still belongs in + // the list — that it exists is the half `manifest.json` + // could not answer. + .map(|c| match c.column_type.is_empty() { + true => serde_json::json!({ "name": c.name }), + false => serde_json::json!({ "name": c.name, "type": c.column_type }), + }) + .collect(), + )); + } + } +} + /// dbt's `materialized` mapped onto Windmill's write strategy. /// /// The mapping is exact for the four strategies Windmill has, and deliberately @@ -655,6 +809,9 @@ pub fn ingest_manifest( .map(|(k, v)| (k.clone(), v.description.clone().unwrap_or_default())) .collect::>()) }), + // Filled by `attach_column_index` when the project asked for it: + // the manifest carries declared columns only. + column_schema: None, freshness: node.freshness.clone(), // The transform the graph renders. Capped: a project can hold // thousands of models and this is duplicated per deploy, so a @@ -826,6 +983,16 @@ pub async fn replace_dbt_manifest( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2 + AND script_hash = $3 AND job_id = $4", + workspace_id, + script_path, + script_hash, + job_id + ) + .execute(&mut **tx) + .await?; // The marker, before the rows: a graph with no nodes at all is a legitimate // answer for a dynamic run that disabled every model, and the reader must be // able to tell it from a run that stored nothing. @@ -881,7 +1048,7 @@ async fn insert_graph_rows( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, \ resource_type, name, asset_path, materialized, materialize_strategy, unique_key, \ tags, description, test_kind, test_column, test_args, severity, attached_node, \ - columns, freshness, raw_code, original_file_path) ", + columns, column_schema, freshness, raw_code, original_file_path) ", ); q.push_values(chunk, |mut b, n| { b.push_bind(workspace_id) @@ -903,6 +1070,7 @@ async fn insert_graph_rows( .push_bind(&n.severity) .push_bind(&n.attached_node) .push_bind(&n.columns) + .push_bind(&n.column_schema) .push_bind(&n.freshness) .push_bind(&n.raw_code) .push_bind(&n.original_file_path); @@ -926,6 +1094,26 @@ async fn insert_graph_rows( q.push(" ON CONFLICT DO NOTHING"); q.build().execute(&mut **tx).await?; } + + for chunk in ingested.column_edges.chunks(COLUMN_EDGE_INSERT_CHUNK) { + let mut q = sqlx::QueryBuilder::new( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, \ + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind) ", + ); + q.push_values(chunk, |mut b, e| { + b.push_bind(workspace_id) + .push_bind(script_path) + .push_bind(script_hash) + .push_bind(job_id) + .push_bind(&e.parent_unique_id) + .push_bind(&e.parent_column) + .push_bind(&e.child_unique_id) + .push_bind(&e.child_column) + .push_bind(&e.lineage_kind); + }); + q.push(" ON CONFLICT DO NOTHING"); + q.build().execute(&mut **tx).await?; + } Ok(()) } @@ -966,7 +1154,7 @@ pub async fn replace_dbt_editor_graph( ) -> Result<()> { // By job alone, so re-executing one — a zombie recovered onto another // worker — replaces its rows rather than colliding with them. - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = $2 AND script_hash IS NULL" )) @@ -1016,7 +1204,7 @@ pub async fn replace_dbt_editor_graph( .fetch_all(&mut **tx) .await?; if !retired.is_empty() { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = ANY($2) \ AND script_hash IS NULL" @@ -1035,6 +1223,8 @@ pub async fn replace_dbt_editor_graph( const NODE_INSERT_CHUNK: usize = 2000; /// Six columns, so the same ceiling allows far more. const EDGE_INSERT_CHUNK: usize = 8000; +/// Nine columns, and by far the most numerous rows of the three. +const COLUMN_EDGE_INSERT_CHUNK: usize = 6000; /// Clear one VERSION's graph: the delete-by-hash route, which only soft-deletes /// its `script` row and so fires no cascade, and the ingest that finds no @@ -1070,6 +1260,15 @@ pub async fn clear_dbt_manifest_version( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + workspace_id, + script_path, + script_hash + ) + .execute(&mut **tx) + .await?; // The marker too, and every job's: a marker left standing for rows that are // gone is read as a snapshot, and its digest still answers the suppression // check — so an identical run would write nothing and then render an empty @@ -1105,7 +1304,7 @@ pub async fn clear_dbt_editor_graphs( workspace_id: &str, script_path: &str, ) -> Result<()> { - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL" @@ -1118,22 +1317,27 @@ pub async fn clear_dbt_editor_graphs( Ok(()) } -/// Move a dbt script's saved retry state to its new path. +/// Move a dbt script's saved state to its new path: the run `dbt retry` resumes, +/// and the state each environment's deferrals resolve through. /// -/// Keyed by path like the sidecar, but unlike the sidecar it is not -/// regenerated by anything: the deploy re-ingests a manifest, while these are -/// the results of a run that already happened. Clearing on rename would throw -/// away a resumable failure for a cosmetic change, so it travels instead. +/// Keyed by path like the sidecar, but unlike the sidecar neither is regenerated +/// by anything: the deploy re-ingests a manifest, while these are the results of +/// runs that already happened. Clearing on rename would throw away a resumable +/// failure, and every deferral until the next full run, for a cosmetic change — +/// so they travel instead. An artifact too large for its row is unaffected: its +/// key is that publication's own, and the moved row is what names it. /// /// See the mutator contract above: this authorizes nothing. -pub async fn move_dbt_run_state( +pub async fn move_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, old_path: &str, new_path: &str, ) -> Result<()> { // The destination may already hold state from a script that lived there - // before; the incoming row is the newer truth for this project. + // before; the incoming row is the newer truth for this project. What the + // displaced row named in object storage is left there, as a cleared one's is + // — see `clear_dbt_script_state`. sqlx::query!( "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2", workspace_id, @@ -1149,22 +1353,38 @@ pub async fn move_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + new_path + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE dbt_environment_state SET script_path = $3 + WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + old_path, + new_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop the saved retry state, but only once NO live version of the path is -/// left. +/// Drop the saved state, but only once NO live version of the path is left. /// -/// `dbt_run_state`'s key is the path and the principal — one saved run per script -/// per identity it executes as, not -/// one per version — so archiving or deleting a single version must not take it -/// with them: the live version's `dbt retry` would be refused and the -/// partial-failure resume lost. It does not need to be version-scoped either, -/// because `identity` already refuses a resume whose project, warehouse or -/// engine moved. +/// Neither table is keyed by version — `dbt_run_state` by path and principal, +/// `dbt_environment_state` by path and environment — so archiving or deleting a +/// single version must not take them with it: the live version's `dbt retry` +/// would be refused, its partial-failure resume lost, and every deferral would +/// have to wait for another full run to republish. Neither needs to be +/// version-scoped either: `identity` already refuses a resume whose project, +/// warehouse or engine moved, and a deferral resolves relation names, which a +/// new version of the same project spells the same way. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state_if_path_retired( +pub async fn clear_dbt_script_state_if_path_retired( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1179,17 +1399,34 @@ pub async fn clear_dbt_run_state_if_path_retired( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2 + AND NOT EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false)", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop a dbt script's saved retry state. +/// Drop a dbt script's saved state, both halves. /// -/// Archive and delete: `run_results` is not small, the invocation arguments it -/// carries are the user's, and a script later created at the same path would -/// otherwise inherit a stranger's resumable failure. +/// Archive and delete: neither is small, the invocation arguments and manifest +/// they carry are the user's, and a script later created at the same path would +/// otherwise inherit a stranger's resumable failure and defer to a project it +/// has nothing to do with. +/// +/// An artifact too large for its row lives in the instance's object storage, and +/// this leaves it there — as a deleted script leaves its bundle. Reaching it from +/// here would mean an object-store client in this crate and a delete that has to +/// land after the caller's transaction commits, for one object per environment of +/// a script that is gone. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state( +pub async fn clear_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1201,6 +1438,13 @@ pub async fn clear_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } @@ -1765,6 +2009,69 @@ mod tests { assert_eq!(back.assets.len(), ingested.assets.len()); assert_eq!(back.assets[0].path, ingested.assets[0].path); } + + // The index describes the whole PROJECT while the graph describes what this + // script's selection builds, so an edge whose endpoint the graph does not + // hold has nothing to draw and must not be stored. + #[test] + fn column_lineage_is_scoped_to_the_nodes_the_graph_kept() { + let mut i = ingested(); + let kept = "model.jaffle_shop.customers"; + let dropped = "model.other_project.elsewhere"; + i.attach_column_index(ColumnIndex { + edges: vec![ + edge("model.jaffle_shop.orders_daily", "id", kept, "id", "copy"), + edge(dropped, "id", kept, "id", "copy"), + edge(kept, "id", dropped, "id", "copy"), + ], + columns: [ + ( + kept.to_string(), + vec![ + col("total", "Float64", 1), + col("id", "Int32", 0), + col("untyped", "", 2), + ], + ), + (dropped.to_string(), vec![col("id", "Int32", 0)]), + ] + .into(), + }); + assert_eq!( + i.column_edges + .iter() + .map(|e| (e.parent_unique_id.as_str(), e.child_unique_id.as_str())) + .collect::>(), + vec![("model.jaffle_shop.orders_daily", kept)] + ); + // In `column_index` order, and a column the analysis could not type still + // belongs to the relation. + assert_eq!( + node(&i, kept).column_schema, + Some(serde_json::json!([ + {"name": "id", "type": "Int32"}, + {"name": "total", "type": "Float64"}, + {"name": "untyped"}, + ])) + ); + assert!(node(&i, "model.jaffle_shop.orders_daily") + .column_schema + .is_none()); + } + + fn edge(from: &str, from_col: &str, to: &str, to_col: &str, kind: &str) -> IngestedColumnEdge { + IngestedColumnEdge { + parent_unique_id: from.into(), + parent_column: from_col.into(), + child_unique_id: to.into(), + child_column: to_col.into(), + lineage_kind: kind.into(), + } + } + + fn col(name: &str, column_type: &str, index: i64) -> IndexedColumn { + IndexedColumn { name: name.into(), column_type: column_type.into(), index } + } } /// Record one model's state for THIS RUN. diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 0d5ebdc2c3..da9f5f3df2 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -272,7 +272,10 @@ fn format_db_error(message: &str, detail: Option<&str>, hint: Option<&str>) -> S msg } -fn error_source_chain(e: &dyn std::error::Error) -> String { +/// `e` followed by each of its sources, `: `-separated. The result is meant for +/// users, and a `reqwest::Error` renders its request URL: never pass one built +/// from a URL carrying credentials in its userinfo. +pub fn error_source_chain(e: &dyn std::error::Error) -> String { let mut msg = e.to_string(); let mut source = e.source(); while let Some(cause) = source { diff --git a/backend/windmill-common/src/folders.rs b/backend/windmill-common/src/folders.rs index 2a6ba935b6..fa4dfdf3b7 100644 --- a/backend/windmill-common/src/folders.rs +++ b/backend/windmill-common/src/folders.rs @@ -78,8 +78,6 @@ pub async fn resolve_folder_default_on_behalf_of( else { return Ok(None); }; - // Uncached: this pair is written straight onto the runnable, where a stale address would - // contradict the principal it is stored beside. let email = crate::users::get_email_from_permissioned_as_uncached(&permissioned_as, w_id, db).await?; Ok(Some((email, permissioned_as))) diff --git a/backend/windmill-common/src/git_sync_oss.rs b/backend/windmill-common/src/git_sync_oss.rs index 5f8ba771fb..4c7de4fa7b 100644 --- a/backend/windmill-common/src/git_sync_oss.rs +++ b/backend/windmill-common/src/git_sync_oss.rs @@ -1,11 +1,16 @@ #[cfg(feature = "private")] #[allow(unused)] pub use crate::git_sync_ee::*; -#[cfg(not(feature = "private"))] +#[cfg(not(all(feature = "private", feature = "enterprise")))] use sqlx::{Pool, Postgres}; use url::Url; -#[cfg(not(feature = "private"))] +/// Gated on the pair to match [`with_stored_credential`] below, whose callers +/// reach it through this facade un-gated and so depend on it. Nothing routes +/// here today (the one caller imports the enterprise item directly), so this is +/// for uniformity: the next plain caller would otherwise find no definition +/// under `private` without `enterprise`. +#[cfg(not(all(feature = "private", feature = "enterprise")))] pub async fn get_github_app_token_internal( _db: &Pool, _job_token: &str, @@ -15,6 +20,21 @@ pub async fn get_github_app_token_internal( )); } +/// Server-held git credentials are an enterprise feature, so on this build a +/// repository URL authenticates with whatever it already carries. +/// +/// Gated on the pair rather than on `private` alone: `private` does not imply +/// `enterprise`, and the callers are plain (no `#[cfg]`), so a build with one +/// and not the other would find neither this nor the enterprise definition. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn with_stored_credential( + _db: &Pool, + _w_id: &str, + url: String, +) -> crate::error::Result { + Ok(url) +} + lazy_static::lazy_static! { /// Matches a `user:password@` (or `user@`) userinfo component right after the URL scheme. static ref GIT_URL_USERINFO_RE: regex::Regex = diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 8180e820f2..ef63fc2347 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -84,6 +84,11 @@ pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth"; // windmill-worker/src/ssh_executor_ee.rs. pub const SSH_EXECUTION_SETTING: &str = "ssh_execution_enabled"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; +/// Whether the instance object store stands in for a workspace without storage of its own +/// as the place its members' AI sessions are backed up to. On unless the row says `false`; +/// inert without an instance object store. +pub const AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING: &str = + "ai_sessions_instance_storage_fallback"; /// Compile a newly deployed script's binary right after its dependency job and push it /// to the instance object store, so the first run does not pay the compile. Inert unless /// instance object storage is configured — without it the binary would only ever land in @@ -118,6 +123,7 @@ pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; pub const OTEL_TRACES_RETENTION_SECS_SETTING: &str = "otel_traces_retention_secs"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const HTTP_ROUTE_WORKSPACED_ROUTE_SETTING: &str = "http_route_workspaced_route"; +pub const HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING: &str = "http_route_default_allowed_origins"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version"; pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; @@ -125,6 +131,112 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Instance-wide announcement rendered above every page of the app (maintenance +/// windows, incidents). Readable by any authenticated user, unlike most settings: +/// the banner exists to be shown to everyone, so it must never hold anything the +/// whole instance may not see. +pub const INSTANCE_BANNER_SETTING: &str = "instance_banner"; + +/// Ceiling on the banner message. The banner is a one-or-two-line strip above every +/// page, so anything longer is a layout accident rather than an announcement. +pub const INSTANCE_BANNER_MESSAGE_MAX_LEN: usize = 500; + +/// Ceiling on the banner's link label, which renders as a button inside that same strip. +pub const INSTANCE_BANNER_LINK_LABEL_MAX_LEN: usize = 60; + +/// Validate an [`INSTANCE_BANNER_SETTING`] value. +/// +/// The banner is the one setting rendered to every user of the instance, so its +/// shape is checked at the boundary rather than trusted from the writer: a value +/// that reaches the browser malformed breaks the layout for everyone at once. +/// +/// The link is restricted to http(s) so a stored `javascript:`/`data:` URL can +/// never become the href of an anchor every user sees. +/// +/// Only shapes that would *misrender* are rejected. An enabled banner with no message +/// is left alone deliberately: it renders as nothing, and every write path here runs +/// under the bulk settings save, so rejecting it would fail an admin's whole settings +/// edit — retention, SMTP and all — over a half-typed announcement. +pub fn validate_instance_banner(value: &serde_json::Value) -> Result<(), String> { + let obj = value + .as_object() + .ok_or_else(|| "must be a JSON object".to_string())?; + + // Field types are checked before their contents. Every read below is an `as_str`/ + // `as_bool`, which reports a wrong-typed field as absent — so without this a + // `"link": 123` would skip the URL checks entirely and be stored, and the settings + // form would then throw on it (`link.trim()` on a number) instead of rendering. + for (field, expected, ok) in [ + ( + "enabled", + "a boolean", + obj.get("enabled").is_none_or(|v| v.is_boolean()), + ), + ( + "dismissible", + "a boolean", + obj.get("dismissible").is_none_or(|v| v.is_boolean()), + ), + ( + "message", + "a string", + obj.get("message").is_none_or(|v| v.is_string()), + ), + ( + "severity", + "a string", + obj.get("severity").is_none_or(|v| v.is_string()), + ), + ( + "link", + "a string", + obj.get("link").is_none_or(|v| v.is_string()), + ), + ( + "link_label", + "a string", + obj.get("link_label").is_none_or(|v| v.is_string()), + ), + ] { + if !ok { + return Err(format!("{field} must be {expected}")); + } + } + + for (field, max) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN), + ] { + let len = obj + .get(field) + .and_then(|v| v.as_str()) + .map_or(0, |s| s.chars().count()); + if len > max { + return Err(format!("{field} must be at most {max} characters")); + } + } + + if let Some(severity) = obj.get("severity").and_then(|v| v.as_str()) { + if !matches!(severity, "info" | "warning" | "error") { + return Err("severity must be one of info, warning, error".to_string()); + } + } + + if let Some(link) = obj.get("link").and_then(|v| v.as_str()) { + if !link.trim().is_empty() { + let url = url::Url::parse(link.trim()) + .map_err(|e| format!("link must be an absolute http(s) URL: {e}"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err("link must use the http or https scheme".to_string()); + } + if !url.has_host() { + return Err("link must include a host".to_string()); + } + } + } + + Ok(()) +} /// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. /// @@ -256,6 +368,125 @@ use std::sync::atomic::AtomicBool; lazy_static::lazy_static! { pub static ref HTTP_ROUTE_WORKSPACED_ROUTE: AtomicBool = AtomicBool::new(false); pub static ref DISABLE_PASSWORD_LOGIN: AtomicBool = AtomicBool::new(false); + /// Origins HTTP routes allow cross-origin when they configure none of their + /// own. Empty means unset, which keeps the historical `*`. + pub static ref HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS: arc_swap::ArcSwap> = + arc_swap::ArcSwap::from_pointee(vec![]); +} + +/// Whether an allowlist places no restriction at all. +/// +/// `*` is the explicit "open on purpose" entry, and a route carrying it behaves +/// exactly as an unconfigured one: it is how a route opts out of a stricter +/// instance default, including back into the `wm_headers` escape hatch. +pub fn allows_any_origin(allowed_origins: &[String]) -> bool { + allowed_origins.iter().any(|allowed| allowed == "*") +} + +/// An allowlist is scanned on every request to a restricted route, including +/// the unauthenticated preflight, so its size is a request cost anyone can +/// trigger. +pub const MAX_ALLOWED_ORIGINS: usize = 100; +pub const MAX_ALLOWED_ORIGIN_LEN: usize = 256; + +/// Reject allowlist entries that cannot be compared, stored, or safely allowed. +/// +/// The stored string is only ever an operand: `match_origin` echoes the +/// request's own `Origin` back, never this value, so a malformed entry matches +/// nothing and fails closed. Shapes that merely cannot match are the editor's +/// business to warn about, not this function's to refuse. What is left are the +/// three cases where permissiveness costs something: `null` is what every +/// sandboxed iframe sends, so allowing it would admit any page that can open +/// one; a comma cannot survive the editor's comma-separated field, which would +/// silently split one entry into two and widen the list; and an unbounded list +/// makes every preflight pay for it. +pub fn validate_allowed_origins(allowed_origins: &[String]) -> crate::error::Result<()> { + if allowed_origins.len() > MAX_ALLOWED_ORIGINS { + return Err(crate::error::Error::BadRequest(format!( + "At most {} allowed origins, got {}.", + MAX_ALLOWED_ORIGINS, + allowed_origins.len() + ))); + } + + for origin in allowed_origins { + if origin == "*" { + continue; + } + + let invalid = |reason: &str| { + crate::error::Error::BadRequest(format!( + "Invalid allowed origin '{}': {}.", + origin, reason + )) + }; + + if origin.is_empty() { + return Err(invalid("must not be empty")); + } + if origin.len() > MAX_ALLOWED_ORIGIN_LEN { + return Err(invalid("is longer than any origin a browser sends")); + } + // The editor edits the whole list as one comma-separated field, so an + // entry carrying a comma comes back as two and widens the list. + if origin.contains(',') { + return Err(invalid("must not contain a comma, which separates entries")); + } + if origin.eq_ignore_ascii_case("null") { + return Err(invalid( + "'null' is what a sandboxed iframe sends, so allowing it would allow any page that can open one", + )); + } + // An Origin header is always visible ASCII, so a value outside it can + // never be the string this is compared against. + if !origin.chars().all(|c| c.is_ascii_graphic()) { + return Err(invalid( + "must contain only visible ASCII, with no whitespace", + )); + } + } + + Ok(()) +} + +/// Read [`HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING`] from its stored value. +/// +/// Accepts the comma-separated string the settings UI writes, or a JSON array +/// for anything setting it through the API directly. +pub fn parse_allowed_origins_setting( + value: Option<&serde_json::Value>, +) -> crate::error::Result> { + let origins = match value { + None | Some(serde_json::Value::Null) => vec![], + Some(serde_json::Value::String(raw)) => raw + .split(',') + .map(|origin| origin.trim().to_string()) + .filter(|origin| !origin.is_empty()) + .collect(), + Some(serde_json::Value::Array(entries)) => entries + .iter() + .map(|entry| match entry { + serde_json::Value::String(origin) => Ok(origin.trim().to_string()), + _ => Err(crate::error::Error::BadRequest(format!( + "{} entries must be strings", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ))), + }) + // Not filtered for empties, unlike the string form: there a + // trailing separator naturally yields an empty token, whereas an + // empty array entry is something the caller wrote and validation + // should reject rather than silently drop. + .collect::>>()?, + Some(_) => { + return Err(crate::error::Error::BadRequest(format!( + "{} expected to be a comma-separated string or an array of strings", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ))) + } + }; + + validate_allowed_origins(&origins)?; + Ok(origins) } pub const ENV_SETTINGS: &[&str] = &[ @@ -327,6 +558,31 @@ pub const ENV_SETTINGS: &[&str] = &[ "OTEL_METRICS", "OTEL_TRACING", "OTEL_LOGS", + // The OTEL_EXPORTER_OTLP_*HEADERS variables are left out: they carry exporter API keys, and + // this list is logged at startup and returned to superadmins by `get_local_settings`. + "OTEL_METRICS_ENABLED", + "OTEL_TRACING_ENABLED", + "OTEL_LOGS_ENABLED", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_COMPRESSION", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "OTEL_METRIC_EXPORT_INTERVAL", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_ENVIRONMENT", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_JOB_LOGS", + "OTEL_TRACES_RETENTION_SECS", + "AI_SHARED_ARTIFACT_RETENTION_SECS", "DISABLE_S3_STORE", "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", @@ -590,6 +846,61 @@ mod tests { } } + #[test] + fn instance_banner_rejects_unsafe_and_malformed_values() { + // The link becomes the href of an anchor shown to every user of the instance, + // so a non-http(s) scheme must not survive a write. + for link in [ + "javascript:alert(1)", + "data:text/html,", + "vbscript:msgbox(1)", + "not-a-url", + "https://", + ] { + let banner = serde_json::json!({ "enabled": true, "message": "down", "link": link }); + assert!( + validate_instance_banner(&banner).is_err(), + "link '{link}' should be rejected" + ); + } + // A wrong-typed field reads as absent to every accessor here, so without an + // explicit type check it would skip validation and be stored. + for bad in [ + serde_json::json!({ "enabled": true, "message": "down", "link": 123 }), + serde_json::json!({ "enabled": true, "message": "down", "link_label": ["a"] }), + serde_json::json!({ "enabled": true, "message": { "text": "down" } }), + serde_json::json!({ "enabled": true, "message": "down", "severity": 2 }), + serde_json::json!({ "enabled": "yes", "message": "down" }), + serde_json::json!({ "enabled": true, "message": "down", "dismissible": "no" }), + ] { + assert!( + validate_instance_banner(&bad).is_err(), + "{bad} should be rejected" + ); + } + // The strip is one or two lines tall; both of its texts are bounded. + for (field, over) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN + 1), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN + 1), + ] { + let mut banner = serde_json::json!({ "enabled": true, "message": "down" }); + banner[field] = serde_json::Value::String("x".repeat(over)); + assert!( + validate_instance_banner(&banner).is_err(), + "an over-long {field} should be rejected" + ); + } + // Enabled with no message renders as nothing and must stay writable: every path + // into this validator is a bulk settings save, so rejecting it would fail an + // admin's unrelated edits over a half-typed announcement. + assert!(validate_instance_banner(&serde_json::json!({ "enabled": true })).is_ok()); + let ok = serde_json::json!({ + "enabled": true, "message": "down", "severity": "warning", + "link": "https://status.example.com", "dismissible": false + }); + assert!(validate_instance_banner(&ok).is_ok()); + } + #[test] fn webhook_base_url_matches_the_ui_validator() { // Kept in lockstep with `isValidWebhookBaseUrl` in diff --git a/backend/windmill-common/src/guest_jwt.rs b/backend/windmill-common/src/guest_jwt.rs index fe05bec6b6..9f3e47d708 100644 --- a/backend/windmill-common/src/guest_jwt.rs +++ b/backend/windmill-common/src/guest_jwt.rs @@ -579,6 +579,15 @@ pub async fn jwks_key_for(url: &str, token: &str) -> Result<(DecodingKey, Vec Result { + // The admit check downstream refuses these anyway; refusing here keeps a deployment + // with no guests from parsing attacker-supplied JWTs at all, and names the reason in + // the log the caller writes. + if !crate::workspaces::instance_supports_guests() { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: {}", + crate::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))); + } if token.len() > MAX_GUEST_JWT_LEN { return Err(Error::NotAuthorized(format!( "guest JWT refused: token is longer than {MAX_GUEST_JWT_LEN} bytes" diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 7ea6dbe5ae..de3fd7684a 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -264,6 +264,8 @@ pub struct GlobalSettings { pub disable_hub: Option, #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_on_deploy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ai_sessions_instance_storage_fallback: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] @@ -916,6 +918,14 @@ pub struct WorkerGroupConfig { pub autoscaling: Option, #[serde(skip_serializing_if = "Option::is_none")] pub native_mode: Option, + /// Object store this group's dependency cache uses instead of the instance one. Same shape + /// as the instance `object_store_cache_config`. + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "instance_config_schema", + schemars(schema_with = "opaque_json_schema") + )] + pub object_store_cache_config: Option, /// Catch-all for fields not yet covered by typed fields. #[serde(flatten)] @@ -1280,8 +1290,9 @@ pub fn diff_worker_configs( ConfigsDiff { upserts, deletes } } -/// Declaratively replace the global settings, rejecting a `github_app_webhook_base_url` -/// the API would reject. +/// Declaratively replace the global settings, rejecting a +/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the +/// API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1330,6 +1341,23 @@ pub async fn sync_global_settings_declarative( } } + let banner_key = crate::global_settings::INSTANCE_BANNER_SETTING; + match desired.get(banner_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(banner) => crate::global_settings::validate_instance_banner(banner) + // The validator's messages name the offending field and its expected type, + // never the submitted value, so they are safe to surface here. + .map_err(|e| anyhow::anyhow!("{banner_key}: {e}"))?, + } + + // An origin list that cannot be parsed is dropped at boot, leaving the + // empty default — which is no restriction at all. Rejecting it here is what + // keeps a typo in a ConfigMap from silently widening CORS instance-wide. + let origins_key = crate::global_settings::HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING; + crate::global_settings::parse_allowed_origins_setting(desired.get(origins_key)) + .map_err(|e| anyhow::anyhow!("{origins_key}: {e}"))?; + let diff = diff_global_settings(current, desired, ApplyMode::Replace); apply_settings_diff(db, &diff).await?; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 7fb5e85a69..8e17ec676e 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -111,6 +111,7 @@ pub use pipeline_advanced_ee as pipeline_advanced; pub use pipeline_advanced_oss as pipeline_advanced; pub mod query_builders; pub mod queue; +pub mod queue_metrics; pub mod result_stream; pub mod runnable_settings; pub mod schedule; @@ -150,6 +151,7 @@ pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans +pub const DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = 60 * 60 * 24 * 30; pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; /// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower @@ -228,6 +230,13 @@ pub fn service_log_retention_secs() -> i64 { SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) } +/// How long a shared AI session artifact stays viewable, in seconds, counted from the last time +/// its author shared it. Read by both the API, which stops serving an expired share, and the +/// monitor, which deletes it — so both must agree, which is why they share this one reader. +pub fn ai_shared_artifact_retention_secs() -> i64 { + *AI_SHARED_ARTIFACT_RETENTION_SECS +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -281,14 +290,16 @@ pub fn check_on_behalf_of_preservation( None } -/// Resolves the identity to store when creating/updating a flow or script. +/// Resolves the identity to store when creating/updating a flow, script or app. /// -/// The permissioned_as is the only stored identity — it decides what the job may access, -/// and the address is derived from it at read time — so the two can never name different -/// accounts. Callers may supply either: a bare email (every client written before the -/// principal existed) is resolved to the principal it names, and an email that names -/// nobody is rejected rather than recorded, since it could only produce a runnable that -/// cannot authenticate. +/// The permissioned_as is the identity: it decides what the job may access, and the address is +/// a function of it, so the two can never name different accounts. For a script or flow the +/// address is derived at read time; an app still stores it, as a compatibility copy written +/// through from the principal on every save and returned verbatim by the app reads (see +/// `docs/app-policy-email-removal.md`). Callers may supply either: a bare email (every client +/// written before the principal existed) is resolved to the principal it names, and an email +/// that names nobody is rejected rather than recorded, since it could only produce a runnable +/// that cannot authenticate. /// /// Returns `None` when the runnable has no on-behalf-of identity, and the caller's own /// identity when they are not allowed to preserve someone else's. @@ -296,6 +307,18 @@ pub fn check_on_behalf_of_preservation( /// Resolves through the non-RLS pool and authorizes nothing itself — `authed` decides only /// whether preservation is allowed, and its role flags are not re-checked against `w_id`. /// Callers must already be authorized for the workspace they pass. +/// +/// Known, accepted race. The lookup runs on the pool, outside the caller's write transaction, so +/// an account renamed or removed between the two has its sweep run before the write is visible, +/// and the write stores the old principal. The runnable then fails to authenticate until it is +/// deployed with a current identity, with two exceptions: an app naming an external superadmin +/// keeps running as that account through its stored address, and if the freed username is later +/// given to another account, the stale principal binds to that account and runs as it. Every +/// caller shares this (scripts, flows and apps, address-only inputs included), and it needs a +/// rename or removal of the exact account inside the lookup-to-commit gap. Closing it means +/// serializing every identity write against every identity mutation, across all runnable kinds +/// (a `usr` row lock in each write, with each sweep ordered after the account change), which no +/// single caller can do on its own; it is left open deliberately. pub async fn resolve_on_behalf_of( on_behalf_of_email: Option<&str>, on_behalf_of: Option<&str>, @@ -461,6 +484,15 @@ lazy_static::lazy_static! { /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the /// only reader. static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); + /// Read it with [`ai_shared_artifact_retention_secs`]. + static ref AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = clamp_retention_secs( + std::env::var("AI_SHARED_ARTIFACT_RETENTION_SECS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS), + DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS, + "AI shared artifact", + ); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); @@ -1868,11 +1900,9 @@ pub async fn on_behalf_of_from_permissioned_as( let Some(permissioned_as) = permissioned_as else { return Ok(None); }; - // Uncached: the address is copied onto the job row, where it stays for the life of the run - // and decides the superadmin flag and the instance groups. Nothing evicts the cache across - // processes, so a cached read would keep minting jobs under an address the account no longer - // holds for up to a minute after it moves. - let email = users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db).await?; + // Cached on purpose, up to one notify poll stale: the accepted dispatch case + // `get_email_from_permissioned_as` documents. + let email = users::get_email_from_permissioned_as(permissioned_as, w_id, db).await?; Ok(Some(jobs::OnBehalfOf { email, permissioned_as: permissioned_as.to_string(), diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs index 2656ce5af5..2008e1ce93 100644 --- a/backend/windmill-common/src/queue.rs +++ b/backend/windmill-common/src/queue.rs @@ -15,6 +15,57 @@ pub async fn get_queue_counts(db: &Pool) -> HashMap { .unwrap_or_else(|| HashMap::new()) } +/// Backlog of a single tag: jobs waiting more than 3 seconds past their `scheduled_for`. +pub struct QueueStat { + pub count: u32, + /// How long the job that would be picked up next has already been waiting, in seconds. + pub delay: f64, + /// When that job started waiting (its `scheduled_for`), in epoch seconds. + pub head_since: f64, +} + +/// Same backlog as [`get_queue_counts`], plus the delay of the job at the head of each +/// tag's queue. The head is picked with the same ordering the worker pull uses, so the +/// delay reported is the one a worker is about to observe. +/// +/// Reads the queue of every workspace: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_counts` does. Unlike [`get_queue_counts`], a failed +/// read is an error rather than an empty map, which would read as every backlog draining. +pub async fn get_queue_stats( + db: &Pool, +) -> crate::error::Result> { + // Grouping by (tag, priority) first finds every head in the same single pass as the + // count. A per-tag `ORDER BY ... LIMIT 1` walks `queue_sort_v2`, whose `tag` column comes + // last, through every other tag's backlog queued ahead of it. + let rows = sqlx::query!( + "SELECT tag AS \"tag!\", count AS \"count!\", + EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\", + EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\" + FROM ( + SELECT tag, sum(n)::bigint AS count, + (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head + FROM ( + SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head + FROM v2_job_queue WHERE + scheduled_for <= now() - ('3 seconds')::interval AND running = false + GROUP BY tag, priority + ) g + GROUP BY tag + ) t", + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|x| { + ( + x.tag, + QueueStat { count: x.count as u32, delay: x.delay, head_since: x.head_since }, + ) + }) + .collect()) +} + pub async fn get_queue_running_counts(db: &Pool) -> HashMap { sqlx::query!( "SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE diff --git a/backend/windmill-common/src/queue_metrics.rs b/backend/windmill-common/src/queue_metrics.rs new file mode 100644 index 0000000000..f5affd7346 --- /dev/null +++ b/backend/windmill-common/src/queue_metrics.rs @@ -0,0 +1,482 @@ +//! The queue metrics the monitor samples into `metrics` (`queue_count_{tag}` and +//! `queue_delay_{tag}`), and how a stored series is drawn back. +//! +//! A stored value is a number, held until the next sample, or, for a delay, `{"since": }`: the job at the head of the queue has been waiting since then and was still there +//! when sampled, so the delay climbs one second per second until the next sample. Besides +//! [`QueueSample`], the SQL in [`read_queue_metrics_series`] and in `GET /workers/queue_metrics` +//! decodes both shapes. + +use std::collections::BTreeMap; + +use serde::Serialize; +use sqlx::{Pool, Postgres}; + +pub const QUEUE_COUNT_PREFIX: &str = "queue_count_"; +pub const QUEUE_DELAY_PREFIX: &str = "queue_delay_"; + +/// A backlogged tag whose value has not moved is re-sampled only this often. A longer heartbeat +/// writes fewer rows, but keeps a tag whose drain was never recorded (no server was up when it +/// drained) drawn as backlogged for longer. +pub const QUEUE_METRIC_HEARTBEAT_SECS: f64 = 5.0 * 60.0; + +/// A series silent for longer than this has drained: the sampler stops looking for it, so no +/// closing zero will come, and it is drawn as zero from there. Heartbeats land up to a monitor +/// tick and a sampling slot late, so this must stay well above their real spacing. +pub const QUEUE_METRIC_STALE_SECS: f64 = 3.0 * QUEUE_METRIC_HEARTBEAT_SECS; + +/// Heads that started waiting within this of each other are one wait: jobs queued together +/// leave the head one after another without the delay dropping. +pub const QUEUE_DELAY_SAME_HEAD_SECS: f64 = 1.0; + +/// Slots a series is split into, whatever the window. A slot draws at most four vertices, and a +/// climb one more at each slot boundary it crosses, so a line stays under about 600 points +/// however many rows the window holds. +const QUEUE_METRICS_SERIES_SLOTS: f64 = 120.0; + +/// A stored sample, as it is drawn from the moment it was written until the next one. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum QueueSample { + /// A count, or a delay while the head keeps changing, which hovers. + Held(f64), + /// A delay while the job that started waiting at `since` (epoch seconds) stays at the head. + Climbing { since: f64 }, +} + +impl QueueSample { + pub fn parse(value: &serde_json::Value) -> Option { + match value.get("since") { + Some(since) => since.as_f64().map(|since| Self::Climbing { since }), + None => value.as_f64().map(Self::Held), + } + } + + pub fn to_json(self) -> serde_json::Value { + match self { + Self::Held(value) => serde_json::json!(value), + Self::Climbing { since } => serde_json::json!({ "since": since }), + } + } + + /// Its value at `t`, in epoch seconds. + pub fn value_at(self, t: f64) -> f64 { + match self { + Self::Held(value) => value, + Self::Climbing { since } => t - since, + } + } + + /// When the job at the head of a delay sample written at `at` started waiting. + pub fn head_since(self, at: f64) -> f64 { + match self { + Self::Held(delay) => at - delay, + Self::Climbing { since } => since, + } + } +} + +#[derive(Serialize)] +pub struct QueueMetricsSeries { + /// The window drawn, in epoch milliseconds. + pub from: i64, + pub to: i64, + pub tags: Vec, +} + +#[derive(Serialize)] +pub struct QueueTagSeries { + pub tag: String, + /// Vertices `[epoch ms, value]` of a line joined by straight segments. + pub count: Vec<(i64, f64)>, + pub delay: Vec<(i64, f64)>, +} + +/// The queue metrics of the last `window_secs`, each series aggregated per slot by the database +/// and drawn by [`render_series`], so the size is bounded by the number of tags rather than by +/// how many rows they wrote. +/// +/// Reads the metrics of every workspace's tags: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_metrics_series` does. +pub async fn read_queue_metrics_series( + db: &Pool, + window_secs: f64, +) -> crate::error::Result { + let to = sqlx::query_scalar!("SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"") + .fetch_one(db) + .await?; + let from = to - window_secs; + let slot_secs = window_secs / QUEUE_METRICS_SERIES_SLOTS; + + // Slot -1 holds the samples written before the window, of which only the last is used: it + // sets the value in force at the left edge. A series silent for longer than the stale window + // reads as zero, so nothing older can matter. Arrays compare element by element, so + // `max(ARRAY[t, v])` is the slot's latest sample, found without sorting every row. `v` is a + // sample's value when it was written: for a climbing delay, how long its head had waited. + // + // A climb keeps rising until the next sample, so when that sample lands in the same slot + // (the tag drained, or its head moved), the climb's top is higher than any `v`. Looking the + // next sample up for the slot's last climb, rather than ordering every row, keeps the pass a + // plain aggregate; an earlier climb in the same slot still shows up to its last heartbeat. + // `t` round-trips through `to_timestamp` to within a microsecond either way, so both bounds + // carry a millisecond of slack, far less than two distinct samples of a series are apart: + // without it the climbing sample can match itself, or the one at `last` fall outside. + let rows = sqlx::query!( + "WITH slots AS ( + SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak, + (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value, + (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing, + COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp, + max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb + FROM ( + SELECT id, t, + CASE jsonb_typeof(value) + WHEN 'number' THEN value::double precision + WHEN 'object' THEN t - (value->>'since')::double precision + END AS v, + (value->>'since')::double precision AS since, + (jsonb_typeof(value) = 'object')::int::double precision AS climbing, + greatest(floor((t - $1::double precision) / $2::double precision), -1)::int + AS slot + FROM ( + SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t + FROM metrics + WHERE id LIKE 'queue_%' + AND created_at > to_timestamp($1::double precision - $3::double precision) + ) m + ) s + WHERE v IS NOT NULL + GROUP BY id, slot + ) + SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\", + greatest(peak, CASE WHEN last_climb[1] < last THEN ( + SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision + FROM metrics n + WHERE n.id = slots.id AND n.id LIKE 'queue_%' + AND n.created_at > to_timestamp(last_climb[1] + 0.001) + AND n.created_at <= to_timestamp(last + 0.001) + ) - last_climb[2] END) AS \"peak!\", + first_value AS \"first_value!\", last_value AS \"last_value!\", + last_climbing AS \"last_climbing!\", ramp AS \"ramp!\" + FROM slots + ORDER BY id, slot", + from, + slot_secs, + QUEUE_METRIC_STALE_SECS, + QUEUE_DELAY_SAME_HEAD_SECS, + ) + .fetch_all(db) + .await?; + + #[derive(Default)] + struct Stored { + carried: Option, + slots: Vec, + } + // [count, delay] per tag. + let mut stored: BTreeMap = BTreeMap::new(); + for row in rows { + let (series, tag) = if let Some(tag) = row.id.strip_prefix(QUEUE_COUNT_PREFIX) { + (0, tag) + } else if let Some(tag) = row.id.strip_prefix(QUEUE_DELAY_PREFIX) { + (1, tag) + } else { + continue; + }; + let series = &mut stored.entry(tag.to_string()).or_default()[series]; + let slot = MetricSlot { + first: row.first, + last: row.last, + peak: row.peak, + first_value: row.first_value, + last_value: row.last_value, + last_climbing: row.last_climbing, + ramp: row.ramp, + }; + if row.slot < 0 { + series.carried = Some(slot); + } else { + series.slots.push(slot); + } + } + + let tags = stored + .into_iter() + .map(|(tag, [count, delay])| { + let draw = + |s: &Stored| render_series(s.carried.as_ref(), &s.slots, from, to, slot_secs); + QueueTagSeries { count: draw(&count), delay: draw(&delay), tag } + }) + // A tag that drained before the window has nothing to draw in it. + .filter(|s| s.count.iter().chain(&s.delay).any(|(_, v)| *v != 0.0)) + .collect(); + + Ok(QueueMetricsSeries { + from: (from * 1000.0).round() as i64, + to: (to * 1000.0).round() as i64, + tags, + }) +} + +/// The stored samples of one series that fall in one time slot. +#[derive(Debug, Clone, Copy)] +pub struct MetricSlot { + /// When the first and the last sample of the slot were written, in epoch seconds. + pub first: f64, + pub last: f64, + /// The highest value the series drew over the slot, a climb that ends inside it included. + pub peak: f64, + pub first_value: f64, + /// The value of the last sample, which holds (or climbs, for a climbing delay) until the next. + pub last_value: f64, + pub last_climbing: bool, + /// Every sample of the slot climbs from the same head, so the slot is one exact ramp. + pub ramp: bool, +} + +/// Draw a stored series over `[from, to]` (epoch seconds), split into slots of `slot_secs`, as +/// the vertices of a line joined by straight segments, each `(epoch ms, value)`. +/// +/// A sample holds its value, or a climbing delay keeps climbing, until the next sample or until +/// the series has been silent for [`QUEUE_METRIC_STALE_SECS`]. `carried` is the slot before +/// `from`, whose last sample sets the left edge. A slot draws its peak across the span of its +/// samples, so a spike shorter than a slot still shows at full height, unless it is a single +/// climb, drawn exactly. A climb gets a vertex at every slot boundary it crosses: the delay axis +/// is logarithmic, so one straight segment across many slots would misplace it. +pub fn render_series( + carried: Option<&MetricSlot>, + slots: &[MetricSlot], + from: f64, + to: f64, + slot_secs: f64, +) -> Vec<(i64, f64)> { + let mut line = Line { points: vec![], from, slot_secs }; + let mut held = carried + .map(Held::after) + .filter(|h| from - h.at <= QUEUE_METRIC_STALE_SECS); + if let Some(h) = held { + line.push(from, h.value_at(from)); + } + for slot in slots { + let entering = line.advance(&mut held, slot.first); + line.push(slot.first, entering); + if slot.ramp { + line.push(slot.first, slot.first_value); + } else { + line.push(slot.first, slot.peak); + line.push(slot.last, slot.peak); + } + line.push(slot.last, slot.last_value); + held = Some(Held::after(slot)); + } + if !line.points.is_empty() { + let value = line.advance(&mut held, to); + line.push(to, value); + } + line.points +} + +/// The last sample drawn: when it was written, its value then, and whether it climbs from there. +#[derive(Clone, Copy)] +struct Held { + at: f64, + value: f64, + climbing: bool, +} + +impl Held { + fn after(slot: &MetricSlot) -> Self { + Self { at: slot.last, value: slot.last_value, climbing: slot.last_climbing } + } + + fn value_at(self, t: f64) -> f64 { + if self.climbing { + self.value + (t - self.at) + } else { + self.value + } + } +} + +struct Line { + points: Vec<(i64, f64)>, + from: f64, + slot_secs: f64, +} + +impl Line { + /// The value `held` has at `t`, drawing the climb that leads there and, when the series went + /// silent for too long first, its drop to zero, after which it is forgotten. + fn advance(&mut self, held: &mut Option, t: f64) -> f64 { + let Some(h) = *held else { + return 0.0; + }; + let stale_at = h.at + QUEUE_METRIC_STALE_SECS; + if h.climbing { + let end = t.min(stale_at); + let start = h.at.max(self.from); + let mut boundary = self.from + + ((start - self.from) / self.slot_secs).floor() * self.slot_secs + + self.slot_secs; + while boundary < end { + self.push(boundary, h.value_at(boundary)); + boundary += self.slot_secs; + } + } + if t <= stale_at { + return h.value_at(t); + } + self.push(stale_at, h.value_at(stale_at)); + self.push(stale_at, 0.0); + *held = None; + 0.0 + } + + fn push(&mut self, t: f64, value: f64) { + let point = ((t * 1000.0).round() as i64, value); + match self.points.as_mut_slice() { + [.., last] if *last == point => {} + // A horizontal run only needs its two ends. + [.., a, b] if a.1 == value && b.1 == value => b.0 = point.0, + _ => self.points.push(point), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const FROM: f64 = 1_000_000.0; + const TO: f64 = FROM + 3600.0; + const SLOT: f64 = 30.0; + + fn held(first: f64, last: f64, peak: f64, last_value: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak, + first_value: peak, + last_value, + last_climbing: false, + ramp: false, + } + } + + /// A slot whose samples all climb from a head that started waiting 30s before `FROM`. + fn climbing(first: f64, last: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak: last + 30.0, + first_value: first + 30.0, + last_value: last + 30.0, + last_climbing: true, + ramp: true, + } + } + + fn at(secs: f64, value: f64) -> (i64, f64) { + (((FROM + secs) * 1000.0) as i64, value) + } + + #[test] + fn a_value_holds_until_the_next_sample_and_a_drain_drops_where_it_was_written() { + let line = render_series( + None, + &[ + held(60.0, 60.0, 3.0, 3.0), + held(600.0, 600.0, 2.0, 2.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + SLOT, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(600.0, 3.0), + at(600.0, 2.0), + at(900.0, 2.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } + + #[test] + fn a_series_silent_past_the_stale_window_drops_to_zero() { + let line = render_series(None, &[held(60.0, 60.0, 3.0, 3.0)], FROM, TO, SLOT); + let dropped = 60.0 + QUEUE_METRIC_STALE_SECS; + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(dropped, 3.0), + at(dropped, 0.0), + at(3600.0, 0.0) + ] + ); + } + + #[test] + fn a_slot_draws_its_peak_then_continues_from_its_last_sample() { + // Samples at 60 (5), 70 (9), 80 (4) collapsed into one slot. + let line = render_series( + Some(&held(-30.0, -30.0, 2.0, 2.0)), + &[held(60.0, 80.0, 9.0, 4.0)], + FROM, + FROM + 120.0, + SLOT, + ); + assert_eq!( + line, + vec![ + at(0.0, 2.0), + at(60.0, 2.0), + at(60.0, 9.0), + at(80.0, 9.0), + at(80.0, 4.0), + at(120.0, 4.0) + ] + ); + } + + #[test] + fn a_climbing_delay_is_drawn_exactly_up_to_its_drain() { + // 300s slots: one holds two climbing samples, and heartbeats follow until the drain. + let line = render_series( + None, + &[ + climbing(60.0, 120.0), + climbing(360.0, 360.0), + climbing(660.0, 660.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + 300.0, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + // The slot is one climb, not its peak held across it. + at(60.0, 90.0), + at(120.0, 150.0), + // A vertex at each slot boundary the climb crosses. + at(300.0, 330.0), + at(360.0, 390.0), + at(600.0, 630.0), + at(660.0, 690.0), + // Still climbing right up to the closing zero. + at(900.0, 930.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } +} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index d71bc50e60..628ae2a709 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -91,9 +91,6 @@ async fn prefetch_cached_script_inner( derive_email: bool, ) -> crate::error::Result> { let derived_email = match script.on_behalf_of.as_deref().filter(|_| derive_email) { - // Uncached: the client preserves this pair and sends it back, where the write path - // validates it against an uncached lookup. A cached address would pair a live principal - // with an address the account no longer holds, and the redeploy would be rejected. Some(permissioned_as) => Some( crate::users::get_email_from_permissioned_as_uncached( permissioned_as, @@ -250,23 +247,37 @@ pub async fn get_full_hub_script_by_path( let version = path_iterator .next() .ok_or_else(|| Error::internal_err(format!("expected hub path to have version number")))?; + // A cache entry that cannot be read or parsed counts as a miss rather than an error: + // a truncated write leaves a file that exists but deserializes to nothing, and refetching + // it is always preferable to failing the job push it was read for. let cache_path = format!("{}/{version}", *HUB_CACHE_DIR); - let script; - if tokio::fs::metadata(&cache_path).await.is_err() { - script = get_full_hub_script_by_path_inner(path, http_client, db).await?; - if let Err(e) = crate::worker::write_file( - &HUB_CACHE_DIR, - &version, - &serde_json::to_string(&script).map_err(to_anyhow)?, - ) { - tracing::error!("failed to write hub script {path} to cache: {e}"); - } else { - tracing::info!("wrote hub script {path} to cache"); + let cached = match tokio::fs::read_to_string(&cache_path).await { + Ok(content) => serde_json::from_str::(&content) + .inspect_err(|e| { + tracing::error!("hub script cache at {cache_path} is unparseable, refetching: {e}") + }) + .ok(), + Err(e) => { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::error!("hub script cache at {cache_path} is unreadable, refetching: {e}"); + } + None } - } else { - let cache_content = tokio::fs::read_to_string(cache_path).await?; - script = serde_json::from_str(&cache_content).unwrap(); + }; + if let Some(script) = cached { tracing::info!("read hub script {path} from cache"); + return Ok(script); + } + + let script = get_full_hub_script_by_path_inner(path, http_client, db).await?; + if let Err(e) = crate::worker::write_file( + &HUB_CACHE_DIR, + &version, + &serde_json::to_string(&script).map_err(to_anyhow)?, + ) { + tracing::error!("failed to write hub script {path} to cache: {e}"); + } else { + tracing::info!("wrote hub script {path} to cache"); } Ok(script) } diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 713b2e0289..31dee749e6 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -8,6 +8,10 @@ pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METAD pub const ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV: &str = "ALLOW_PRIVATE_GUEST_JWKS_URLS"; +/// Lets every git call reach hosts on a private network, whoever it is made for. +/// Without it, [`private_git_host_allowed`] decides. +pub const ALLOW_LOCAL_GIT_REMOTES_ENV: &str = "ALLOW_LOCAL_GIT_REMOTES"; + /// Why a URL failed SSRF validation. /// /// The distinction matters for callers that gate private endpoints behind a @@ -203,6 +207,54 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } +fn allow_local_git_remotes() -> bool { + std::env::var(ALLOW_LOCAL_GIT_REMOTES_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") +} + +/// Who a git call is made for, which decides whether it may reach a host on a +/// private network. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GitRemoteCaller { + /// A workspace admin's request, or Windmill's own work (polling, webhook and + /// token upkeep, the merge request after a deploy), whose errors only admins read. + AdminOrSystem, + /// A request from anyone who is not a workspace admin. + NonAdmin, +} + +/// Whether a git call made for `caller` may reach a host on a private network. +/// +/// The refusal is for non-admins, who may not be able to run code (operators) +/// and would read git's error output back as a probe of the server's network. +/// An admin can run code, which reaches those hosts from a worker already. On a +/// cloud instance, where a workspace admin is anyone who signed up, every caller +/// is refused. +pub fn private_git_host_allowed(caller: GitRemoteCaller) -> bool { + git_host_policy_allows( + caller, + allow_local_git_remotes(), + *crate::worker::CLOUD_HOSTED, + ) +} + +fn git_host_policy_allows(caller: GitRemoteCaller, opted_in: bool, cloud_hosted: bool) -> bool { + opted_in || (caller == GitRemoteCaller::AdminOrSystem && !cloud_hosted) +} + +/// Appended to a refusal of a private git host, naming what would let `caller` +/// through. `None` where nothing an instance administrator sets would help. +pub fn private_git_host_hint(caller: GitRemoteCaller) -> Option { + (caller == GitRemoteCaller::NonAdmin && !*crate::worker::CLOUD_HOSTED).then(|| { + format!( + "Only workspace admins can reach a git server on a private network. To allow \ + every user, set the {ALLOW_LOCAL_GIT_REMOTES_ENV}=true environment variable on \ + the Windmill servers" + ) + }) +} + pub async fn validate_saml_metadata_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -632,6 +684,15 @@ mod tests { )); } + #[test] + fn private_git_hosts_are_refused_to_non_admins_and_on_cloud() { + use GitRemoteCaller::{AdminOrSystem, NonAdmin}; + assert!(git_host_policy_allows(AdminOrSystem, false, false)); + assert!(!git_host_policy_allows(NonAdmin, false, false)); + assert!(!git_host_policy_allows(AdminOrSystem, false, true)); + assert!(git_host_policy_allows(NonAdmin, true, true)); + } + #[tokio::test] async fn saml_ssrf_error_message_includes_env_hint_only_for_private_urls() { let private_error = validate_url_for_ssrf("http://127.0.0.1/metadata") diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 115dae34e7..aed9b5c5de 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -401,6 +401,45 @@ pub async fn fetch_draft_only_list_rows( Ok(rows) } +/// Delete the caller's OWN draft at a path with no deployed row, for the DELETE +/// route of a kind whose list synthesizes such rows via +/// `fetch_draft_only_list_rows`. The `NOT EXISTS` leaves a deployed row's draft +/// alone, so a route may call this on its not-found branch whatever the reason +/// for the miss. `Ok(false)` means nothing matched: the caller reports its own error. +/// +/// Takes no permission check and callers must not add one: an email-scoped row +/// belongs to the caller, who can always discard it, as `update_draft`'s +/// own-discard does. Legacy (`email IS NULL`) rows are owned by nobody and keep +/// their write gate, so discarding one stays on the `update_draft` route. +pub async fn delete_draft_only_for_path( + db: &DB, + w_id: &str, + kind: UserDraftItemKind, + path: &str, + email: &str, +) -> Result { + let Some(table) = kind.deployed_table() else { + return Ok(false); + }; + // `table` is from the closed `deployed_table()` enum, never user input. + let sql = format!( + "DELETE FROM draft \ + WHERE workspace_id = $1 AND typ = $2::text::DRAFT_KIND AND path = $3 \ + AND email = $4 \ + AND NOT EXISTS (SELECT 1 FROM {table} t \ + WHERE t.workspace_id = draft.workspace_id AND t.path = draft.path)" + ); + let deleted = sqlx::query(&sql) + .bind(w_id) + .bind(kind.as_str()) + .bind(path) + .bind(email) + .execute(db) + .await? + .rows_affected(); + Ok(deleted > 0) +} + /// The get-by-path draft choreography, shared by every entity's "get by path" /// route. Given the deployed entity as an `Option` (caller maps its own "not /// found" to `None`): diff --git a/backend/windmill-common/src/usernames.rs b/backend/windmill-common/src/usernames.rs index 6fde2a8186..adb81ae682 100644 --- a/backend/windmill-common/src/usernames.rs +++ b/backend/windmill-common/src/usernames.rs @@ -17,6 +17,23 @@ lazy_static::lazy_static! { pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); } +/// Width of the `username` columns of `usr`, `password` and `pending_user`. +pub const USERNAME_MAX_LEN: usize = 50; + +/// `base` with the collision suffix of `attempt` appended (none for the first attempt), cut +/// to `USERNAME_MAX_LEN`. A local part longer than the column is a valid email, and an +/// insert that fails on the derived username rolls back everything around it. +pub fn fit_username(base: &str, attempt: u32) -> String { + let suffix = if attempt > 1 { + attempt.to_string() + } else { + String::new() + }; + let mut username: String = base.chars().take(USERNAME_MAX_LEN - suffix.len()).collect(); + username.push_str(&suffix); + username +} + pub async fn generate_instance_wide_unique_username<'c>( tx: &mut Transaction<'c, Postgres>, email: &str, @@ -41,9 +58,7 @@ pub async fn generate_instance_wide_unique_username<'c>( email ))); } - if i > 1 { - username = format!("{}{}", base_username, i) - } + username = fit_username(&base_username, i); username_conflict = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", &username, @@ -164,3 +179,19 @@ pub async fn get_instance_username_or_create_pending<'c>( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fit_username_keeps_the_column_width() { + assert_eq!(fit_username("alice", 1), "alice"); + assert_eq!(fit_username("alice", 2), "alice2"); + let base = "a".repeat(60); + assert_eq!(fit_username(&base, 1), "a".repeat(USERNAME_MAX_LEN)); + let with_suffix = fit_username(&base, 1000); + assert_eq!(with_suffix.len(), USERNAME_MAX_LEN); + assert!(with_suffix.ends_with("1000")); + } +} diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 3d00c5d312..1b696746b0 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -13,6 +13,35 @@ lazy_static::lazy_static! { pub static ref VALID_EMAIL: regex::Regex = regex::Regex::new( r"^[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+(\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?\.)+[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$" ).unwrap(); + +} + +/// Width of the `email` columns of `usr`, `workspace_invite` and `email_to_igroup`. +pub const EMAIL_COLUMN_MAX_LEN: usize = 255; + +/// The regex of the `proper_email` CHECK constraint on `usr` and `workspace_invite` +/// (`20220620210708_regex_fix`), verbatim, for [`usr_accepts_email`]. Evaluated by the +/// database and never by a Rust engine: `~*` folds case under the database collation, so a +/// fixed mirror accepts addresses the constraint rejects, or rejects ones it holds, on some +/// locale. `windmill-common/tests/usr_accepts_email.rs` pins the text to the constraint. +pub const PROPER_EMAIL_PATTERN: &str = r#"^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$"#; + +/// Whether `usr` (and `workspace_invite`) will store `email`: the `proper_email` regex as the +/// database evaluates it, plus the column width. Unlike [`VALID_EMAIL`] this admits every +/// address those tables already hold, which matters wherever an existing member is judged. +pub async fn usr_accepts_email<'c, E>(db: E, email: &str) -> crate::error::Result +where + E: sqlx::Executor<'c, Database = sqlx::Postgres>, +{ + if email.contains('\0') || email.chars().count() > EMAIL_COLUMN_MAX_LEN { + return Ok(false); + } + let accepted: bool = sqlx::query_scalar("SELECT $1::text ~* $2::text") + .bind(email) + .bind(PROPER_EMAIL_PATTERN) + .fetch_one(db) + .await?; + Ok(accepted) } pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev"; @@ -21,6 +50,13 @@ pub const SUPERADMIN_SYNC_EMAIL: &str = "superadmin_sync@windmill.dev"; pub const COOKIE_NAME: &str = "token"; +/// `password.login_type` of an account created for someone before they have signed in: +/// no credential of its own (password login and reset require `'password'`), reachable +/// only through a superadmin-minted login link until either `set_password` turns it into +/// a password account or the first OAuth login proving the same address adopts it and +/// rewrites `login_type` to the provider. +pub const PENDING_OAUTH_LOGIN_TYPE: &str = "pending_oauth"; + /// Prefix for user-based permissioned_as values: "u/" pub const PERMISSIONED_AS_USER_PREFIX: &str = "u/"; /// Prefix for group-based permissioned_as values: "g/" @@ -175,13 +211,19 @@ pub async fn permissioned_as_exists( /// Drop a cached address so a transactional email change is visible immediately. /// -/// The address is derived at dispatch and feeds the instance-superadmin check and -/// `email_to_igroup`, so serving a stale one would run jobs with the wrong authorization -/// for up to the cache TTL. +/// Not the thing that keeps authorization correct — `fetch_authed_from_permissioned_as` +/// re-resolves the address before granting anything. This keeps the cache from serving an +/// address that is merely wrong for the TTL, on reads and on what is shown. pub fn invalidate_email_cache(workspace_id: &str, username: &str) { EMAIL_CACHE.remove(&(workspace_id.to_string(), username.to_string())); } +/// Drop this name's entry in every workspace, for the changes that know the name but not the +/// workspace: a superadmin resolves through `password`, whose row names no workspace of its own. +pub fn invalidate_email_cache_for_username(username: &str) { + EMAIL_CACHE.retain(|(_workspace_id, cached_username), _| cached_username != username); +} + /// Inverse of [`get_email_from_permissioned_as`]: the principal an on-behalf-of email /// names in this workspace, for callers that supply the email alone. /// @@ -194,6 +236,14 @@ pub fn invalidate_email_cache(workspace_id: &str, username: &str) { /// not a superadmin's, or a group that no longer exists. Callers then leave the identity /// unrecorded rather than storing a principal that cannot authenticate. /// +/// Known, accepted consequence of a real account winning the synthetic `group-*@windmill.dev` +/// namespace: a group identity sent as its address alone, as a "keep target identity" workspace +/// deploy sends it for scripts, flows and apps, comes back as the account holding that address +/// when one exists, not as `g/*`. Such an account takes an admin to exist: a superadmin or an +/// admin-configured identity provider to create it (the public OAuth providers only assert a +/// `@windmill.dev` address to that domain's owner) and an admin of the target workspace to admit +/// it, so no member can steer a group's runnables to themselves this way. +/// /// Reads through the non-RLS pool and authorizes nothing: callers must already be authorized /// for `workspace_id`. pub async fn permissioned_as_from_email( @@ -242,6 +292,33 @@ pub async fn permissioned_as_from_email( /// - "u/{username}" → resolve via [`resolve_username_to_email`] (cached) /// - "g/{group}" → "group-{group}@windmill.dev" /// - raw email → return as-is +/// +/// `notify_user_email_change` evicts the key on every process for each change that can move it, +/// at that process's next notify-event poll (`LISTEN_NEW_EVENTS_INTERVAL_SEC`, 10s by default), +/// so a hit can still be the old address for up to one poll. The TTL caps it if an eviction is +/// ever missed. +/// +/// Which of the two to use is a question of how long a wrong answer lives, not of whether it is +/// stored — both of these get stored and read back. A config row (an app policy, a schedule, a +/// runnable) is the authority for every run that follows it, so a stale address there is +/// permanent and invisible: those use [`get_email_from_permissioned_as_uncached`]. Job dispatch +/// also stores its answer, and the worker reads it back to build that run's authed, but it +/// governs one job and dies with it, so it stays here. +/// +/// The job's own authorization does not trust the address as given: +/// `fetch_authed_from_permissioned_as` re-resolves it from the principal's live binding, and that +/// corrected address is what the job row and its token carry. Route an address into an `Authed`, +/// a job row or a token without going through that function, and this cache stops being safe to +/// read at dispatch. +/// +/// What reads the dispatch address before that re-resolution (the quota and superadmin-exemption +/// checks at the top of `push_inner`, a flow step's tag check) or when the principal has no live +/// binding can act on the old address for up to one poll after a username reuse, an email change +/// or a superadmin change. That window is accepted as the cost of keeping dispatch off the +/// database; a consumer that cannot tolerate it must re-resolve first. +/// +/// Reads through the non-RLS pool and authorizes nothing — callers must already be authorized +/// for `workspace_id`. pub async fn get_email_from_permissioned_as<'c>( permissioned_as: &str, workspace_id: &str, @@ -250,13 +327,21 @@ pub async fn get_email_from_permissioned_as<'c>( get_email_from_permissioned_as_inner(permissioned_as, workspace_id, db, true).await } -/// [`get_email_from_permissioned_as`] without the address cache. Nothing evicts that cache -/// across processes, so for a minute after an email change it still serves the old address — -/// fine where the address only labels something on screen, wrong where it decides whether a -/// write is accepted or is copied onto a job row that outlives the window. +/// [`get_email_from_permissioned_as`] for a value about to be **persisted**. /// -/// Reads through the non-RLS pool and authorizes nothing, like the cached one: callers must -/// already be authorized for `workspace_id`. +/// The eviction is delivered by the `notify_event` poller, not synchronously, so for a few +/// seconds after a change a replica can still serve the old address. In a config row that is +/// permanent: the row outlives the eviction, every later run trusts it, and nothing re-derives +/// it, so a principal and an address that name different accounts stay that way. +/// +/// Use this for three cases, all of which end in a stored pair: +/// - writing the address into a row; +/// - the lookup that validates a pair before it is stored; +/// - **reads whose result the client sends back** — a script or a workspace export hands over a +/// principal and address together, and a redeploy validates that pair against a fresh +/// resolution, so a stale one comes back as a rejected deploy rather than a stale display. +/// +/// See [`get_email_from_permissioned_as`] for the dispatch case that deliberately does not. pub async fn get_email_from_permissioned_as_uncached<'c>( permissioned_as: &str, workspace_id: &str, diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 62d47a34d7..4b93bb89ff 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -1125,19 +1125,35 @@ use tokio::time::{self, Duration, Sleep}; use pin_project_lite::pin_project; +/// What a [`WarnAfterFuture`] is timing, which decides how its warning reads. +pub enum WarnSubject { + /// A database query, with the SQL when the caller has it. + Query(Option), + /// Anything else (a child process, a cache transfer), named for the log line. + Step(String), +} + pub trait WarnAfterExt: Future + Sized { /// Warns if the future takes longer than the specified number of seconds to complete. #[track_caller] fn warn_after_seconds(self, seconds: u8) -> WarnAfterFuture { let caller = Location::caller(); - self.build_from_caller(seconds, caller, None) + self.build_from_caller(seconds, caller, WarnSubject::Query(None)) + } + + /// Same, for a step that is not a database query (a child process, a cache transfer): + /// the warning names `step` instead of reporting a slow query. + #[track_caller] + fn warn_after_seconds_for(self, seconds: u8, step: &str) -> WarnAfterFuture { + let caller = Location::caller(); + self.build_from_caller(seconds, caller, WarnSubject::Step(step.to_string())) } fn build_from_caller( self, seconds: u8, caller: &Location, - sql: Option, + subject: WarnSubject, ) -> WarnAfterFuture { let location = format!("{}:{}", caller.file(), caller.line()); WarnAfterFuture { @@ -1147,13 +1163,13 @@ pub trait WarnAfterExt: Future + Sized { start_time: std::time::Instant::now(), location, seconds, - sql, + subject, } } #[track_caller] fn warn_after_seconds_with_sql(self, seconds: u8, sql: String) -> WarnAfterFuture { let caller = Location::caller(); - self.build_from_caller(seconds, caller, Some(sql)) + self.build_from_caller(seconds, caller, WarnSubject::Query(Some(sql))) } } @@ -1171,7 +1187,7 @@ pin_project! { location: String, start_time: std::time::Instant, seconds: u8, - sql: Option, + subject: WarnSubject, } } @@ -1191,12 +1207,20 @@ impl Future for WarnAfterFuture { // Poll the timeout future to check if it has elapsed. if !*this.warned { if this.timeout.poll(cx).is_ready() { - tracing::warn!( - location = this.location, - "SLOW_QUERY: query {} to db taking longer than expected (> {} seconds)", - build_query_string(&this.location, this.sql.as_deref()), - this.seconds, - ); + match &*this.subject { + WarnSubject::Step(step) => tracing::warn!( + location = this.location, + "SLOW_STEP: {step} at {} taking longer than expected (> {} seconds)", + this.location, + this.seconds, + ), + WarnSubject::Query(sql) => tracing::warn!( + location = this.location, + "SLOW_QUERY: query {} to db taking longer than expected (> {} seconds)", + build_query_string(&this.location, sql.as_deref()), + this.seconds, + ), + } *this.warned = true; } } @@ -1206,12 +1230,20 @@ impl Future for WarnAfterFuture { Poll::Ready(output) => { if *this.warned { let elapsed = this.start_time.elapsed(); - tracing::warn!( - location = this.location, - "SLOW_QUERY: completed query {} with total duration: {:.2?}", - build_query_string(&this.location, this.sql.as_deref()), - elapsed - ); + match &*this.subject { + WarnSubject::Step(step) => tracing::warn!( + location = this.location, + "SLOW_STEP: {step} at {} completed with total duration: {:.2?}", + this.location, + elapsed + ), + WarnSubject::Query(sql) => tracing::warn!( + location = this.location, + "SLOW_QUERY: completed query {} with total duration: {:.2?}", + build_query_string(&this.location, sql.as_deref()), + elapsed + ), + } } Poll::Ready(output) } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 045ddcd95b..1feb4182ab 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -252,12 +252,17 @@ pub async fn build_crypt_with_key_suffix( key_suffix: &str, ) -> crate::error::Result { let key = get_workspace_key(w_id, db).await?; + Ok(crypt_from_key_with_suffix(&key, key_suffix)) +} + +/// The cipher `build_crypt_with_key_suffix` builds, from a key string in hand. +pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> MagicCrypt256 { let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { format!("{}{}{}", key, salt, key_suffix) } else { format!("{}{}", key, key_suffix) }; - Ok(magic_crypt::new_magic_crypt!(crypt_key, 256)) + magic_crypt::new_magic_crypt!(crypt_key, 256) } pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result { diff --git a/backend/windmill-common/src/wac.rs b/backend/windmill-common/src/wac.rs index d46c937e33..05c343b427 100644 --- a/backend/windmill-common/src/wac.rs +++ b/backend/windmill-common/src/wac.rs @@ -8,11 +8,17 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::{Postgres, Transaction}; +use std::sync::atomic::AtomicBool; use uuid::Uuid; use crate::error::{self, Error}; use crate::DB; +/// Set when a child completion brought a parked WAC parent's `suspend` counter to +/// zero, so a worker's pull loop tries the suspended-jobs query first instead of +/// waiting for its next periodic attempt. +pub static WAC_SUSPEND_READY: AtomicBool = AtomicBool::new(false); + /// Checkpoint state persisted across workflow invocations. #[derive(Debug, Serialize, Deserialize, Default, Clone)] pub struct WacCheckpoint { @@ -672,3 +678,199 @@ pub async fn persist_inline_checkpoint_delta( Ok(failure) } + +/// The step key a WAC v2 parent is waiting on `child_job` for, if any. +/// +/// `job_ids` is the parent's `pending_steps.job_ids` (step key → child job id). +/// A child absent from it is not a step this round is waiting on: a completion +/// arriving after the parent re-dispatched the key to a new job, or a child the +/// body launched directly (`runScript` and friends). +fn pending_step_key(job_ids: &Value, child_job: &Uuid) -> Option { + let child = child_job.to_string(); + job_ids + .as_object()? + .iter() + .find_map(|(key, id)| (id.as_str() == Some(child.as_str())).then(|| key.clone())) +} + +/// Record a completed child on its WAC parent, in the child's completion transaction. +/// +/// Every path that brings a child to a terminal state — a worker's result, the +/// zombie monitor, a cancel — completes it through `add_completed_job`, and this is +/// where the parent learns of it. For a child the parent is parked on, the step's +/// result (or failure record) is merged into the checkpoint's `completed_steps` and +/// the parent's `suspend` counter drops by one, atomically with the child's own +/// completion: there is no window in which the child is completed but the parent +/// still waits for it. For any other child, only the timeline entry is stamped. +/// +/// Returns whether the counter reached zero, i.e. the parent is ready to be pulled. +/// +/// Exactly once: the merge is refused when `completed_steps` already holds the key +/// or `job_ids` no longer maps it to this child, and the decrement follows only a +/// merge that happened. Two completions of one child (a worker and the monitor +/// racing) therefore decrement once, and a stale completion never touches a +/// counter that belongs to a later round. +/// +/// Lock order: the parent's queue row, then its status row, then (by the caller) +/// the child's queue row. A cancel walks parent then children, the parent's own +/// completion deletes its queue row and cascades to its status row, and the park +/// (`suspend_wac_parent`) locks the queue row before writing the checkpoint, so +/// any other order can deadlock against one of them. +/// +/// Authorization: none is checked here. `child_job` is the job the caller is +/// completing, which it already holds, and `parent_job` must be that job's +/// persisted `v2_job.parent_job` (both callers read it from the child's row). +/// The read that gates the step merge and the counter decrement joins on that +/// relationship, so a mismatched pair changes no parent's `completed_steps` or +/// `suspend`; only the timeline stamp at the end runs unconditionally. Job ids +/// are global, so no workspace scoping is needed on top. +pub async fn record_child_completion( + tx: &mut Transaction<'_, Postgres>, + parent_job: &Uuid, + child_job: &Uuid, + success: bool, + duration_ms: i64, + result: &str, +) -> error::Result { + let job_ids: Option> = sqlx::query_scalar( + "SELECT s.workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ + FROM v2_job_status s JOIN v2_job c ON c.parent_job = s.id \ + WHERE s.id = $1 AND c.id = $2", + ) + .bind(parent_job) + .bind(child_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to read WAC parent {parent_job}: {e}")))?; + + let step_key = job_ids + .flatten() + .and_then(|ids| pending_step_key(&ids, child_job)); + + let mut parent_ready = false; + if let Some(step_key) = step_key { + let parked: Option = + sqlx::query_scalar("SELECT suspend FROM v2_job_queue WHERE id = $1 FOR UPDATE") + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to lock WAC parent {parent_job}: {e}")) + })?; + + if parked.is_some() { + let step_value = if success { + result.to_string() + } else { + let raw: Value = serde_json::from_str(result).unwrap_or(Value::Null); + wac_failure_record(&step_key, Some(&child_job.to_string()), &raw).to_string() + }; + let merged: Option = sqlx::query_scalar( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + workflow_as_code_status, + '{_checkpoint,completed_steps}', + COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) + || jsonb_build_object($2::text, $3::text::jsonb) + ) WHERE id = $1 + AND workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids'->>$2 = $4 + AND NOT COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps' ? $2, false) + RETURNING 1", + ) + .bind(parent_job) + .bind(&step_key) + .bind(&step_value) + .bind(child_job.to_string()) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to add WAC completed step: {e}")) + })?; + + if merged.is_some() { + // `suspend_until` stays set: the suspended pull query is what takes a + // parked parent back, and it selects on `suspend_until IS NOT NULL`. + let suspend: Option = sqlx::query_scalar( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 RETURNING suspend", + ) + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to unsuspend WAC parent: {e}")))?; + parent_ready = suspend == Some(0); + if parent_ready { + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' WHERE id = $1", + ) + .bind(parent_job) + .execute(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to clear WAC pending steps: {e}")) + })?; + } + } + tracing::info!( + parent_job = %parent_job, + child_job = %child_job, + step_key = %step_key, + success, + recorded = merged.is_some(), + parent_ready, + "WAC v2 child job completed" + ); + } + } + + // The child's entry in the parent's timeline, keyed by child id. The parent may + // already be completed (cancelled with its children still running), in which + // case the entry lives on its completed row instead. Errors propagate: a failed + // statement has already aborted the transaction, so there is nothing to continue with. + let stamped: Option = sqlx::query_scalar( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + jsonb_set( + workflow_as_code_status, + ARRAY[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + ARRAY[$1, 'duration_ms'], + to_jsonb($2::bigint) + ) WHERE id = $3 AND workflow_as_code_status IS NOT NULL RETURNING 1", + ) + .bind(child_job.to_string()) + .bind(duration_ms) + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not update parent job `duration_ms` in workflow as code status: {e}" + )) + })?; + if stamped.is_none() { + sqlx::query( + "UPDATE v2_job_completed SET workflow_as_code_status = jsonb_set( + jsonb_set( + workflow_as_code_status, + ARRAY[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + ARRAY[$1, 'duration_ms'], + to_jsonb($2::bigint) + ) WHERE id = $3 AND workflow_as_code_status IS NOT NULL", + ) + .bind(child_job.to_string()) + .bind(duration_ms) + .bind(parent_job) + .execute(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not update completed parent job `duration_ms` in workflow as code status: {e}" + )) + })?; + } + + Ok(parent_ready) +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 84284759c6..ee8a389bb7 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -392,6 +392,7 @@ lazy_static::lazy_static! { pip_local_dependencies: Default::default(), env_vars: Default::default(), native_mode: false, + object_store_cache_config: Default::default(), }); pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); @@ -2343,6 +2344,7 @@ pub async fn load_worker_config( .or_else(|| load_additional_python_paths_from_env()), env_vars: resolved_env_vars, native_mode, + object_store_cache_config: config.object_store_cache_config, }) } @@ -2432,6 +2434,7 @@ pub struct WorkerConfigOpt { pub env_vars_static: Option>, pub env_vars_allowlist: Option>, pub native_mode: Option, + pub object_store_cache_config: Option, } impl Default for WorkerConfigOpt { @@ -2450,6 +2453,7 @@ impl Default for WorkerConfigOpt { env_vars_static: Default::default(), env_vars_allowlist: Default::default(), native_mode: Default::default(), + object_store_cache_config: Default::default(), } } } @@ -2468,12 +2472,18 @@ pub struct WorkerConfig { pub pip_local_dependencies: Option>, pub env_vars: HashMap, pub native_mode: bool, + /// Object store this group's dependency cache uses instead of the instance one, as stored + /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate + /// that parses it, and comparing the raw value is what tells a reload the store changed. + pub object_store_cache_config: Option, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode) + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, object_store_cache_config: {} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, + // holds bucket credentials + self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None")) } } @@ -2502,6 +2512,42 @@ pub fn split_python_requirements>(requirements: T) -> Vec .collect() } +/// Byte offset of the comment marker, per pip's rule: a `#` at line start or preceded by +/// whitespace. A `#` elsewhere belongs to the requirement (`pkg @ https://h/p.whl#sha256=…`). +fn requirement_comment_start(line: &str) -> Option { + line.char_indices() + .find(|(i, c)| *c == '#' && (*i == 0 || line[..*i].ends_with(char::is_whitespace))) + .map(|(i, _)| i) +} + +/// The installable requirement carried by one lockfile line, or `None` for a comment, a +/// `-r`/`-e`/`--flag` directive, or a blank. +/// +/// Windmill installs a lockfile one entry at a time as a `uv pip install` argument, so +/// requirements-file syntax a file-level parser would absorb is an unparseable package name +/// here and has to be stripped first. +pub fn requirement_from_lockfile_line(line: &str) -> Option<&str> { + let requirement = match requirement_comment_start(line) { + Some(i) => &line[..i], + None => line, + } + .trim() + // Continuations are stripped, not joined: right for `--generate-hashes` locks, whose + // continued lines are `--hash=` flags this function drops, but a lock continuing onto a + // marker or extra would lose it. + .trim_end_matches('\\') + .trim_end(); + + (!requirement.is_empty() && !requirement.starts_with('-')).then_some(requirement) +} + +/// Whether a lockfile line continues onto the next one. The continued lines reach the +/// installer as entries of their own rather than being joined, so a caller that cares what +/// they carried — `--hash=` pins, for a `--generate-hashes` lock — has to say so itself. +pub fn lockfile_line_has_continuation(line: &str) -> bool { + line.trim_end().ends_with('\\') +} + #[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] #[repr(u32)] pub enum PyVAlias { @@ -2620,6 +2666,52 @@ mod tests { ids.iter().map(|s| s.to_string()).collect() } + /// Fixtures are verbatim `uv pip compile` output (uv 0.11.28): split and inline + /// annotation styles, and `--generate-hashes`. + #[test] + fn test_requirement_from_lockfile_line() { + assert_eq!(requirement_from_lockfile_line(" # via httpx"), None); + assert_eq!(requirement_from_lockfile_line(" # via"), None); + assert_eq!(requirement_from_lockfile_line(" # anyio"), None); + assert_eq!( + requirement_from_lockfile_line(" # via -r .tmp/requirements.in"), + None + ); + assert_eq!( + requirement_from_lockfile_line("anyio==4.15.1 \\"), + Some("anyio==4.15.1") + ); + assert_eq!( + requirement_from_lockfile_line( + " --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7 \\" + ), + None + ); + assert_eq!(requirement_from_lockfile_line("# py: 3.11"), None); + assert_eq!(requirement_from_lockfile_line("-r other.txt"), None); + assert_eq!( + requirement_from_lockfile_line("--index-url https://x"), + None + ); + assert_eq!(requirement_from_lockfile_line(" "), None); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0"), + Some("httpx==0.27.0") + ); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0 # via -r requirements.in"), + Some("httpx==0.27.0") + ); + // A `#` not preceded by whitespace is part of the requirement, not a comment. + assert_eq!( + requirement_from_lockfile_line("wmill @ https://h/wmill.whl#sha256=abc"), + Some("wmill @ https://h/wmill.whl#sha256=abc") + ); + + assert!(lockfile_line_has_continuation("anyio==4.15.1 \\")); + assert!(!lockfile_line_has_continuation("anyio==4.15.1")); + } + #[test] fn test_parse_job_oom_score_adj() { assert_eq!(parse_job_oom_score_adj(Some("300")), 300); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index b6d10cb7df..7cf910ef84 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -183,7 +183,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28969/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -191,7 +191,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28930/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28957/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -345,13 +345,14 @@ pub struct GitRepositorySettings { #[serde(default, skip_serializing_if = "Option::is_none")] pub auto_pull: Option, /// Open a PR when a deploy pushes a `wm_deploy/**` branch of this promotion - /// repo (app-backed only; runs from the deploy callback so it works without + /// repo (needs a credential the server holds — a GitHub App installation or + /// a checked GitLab token; runs from the deploy callback so it works without /// inbound webhooks). Off by default so upgrades don't change behavior. #[serde(default, skip_serializing_if = "is_false")] pub promotion_open_prs: bool, /// Parent-level: open a PR when a fork of this workspace deploys to its - /// `wm-fork/**` branch (app-backed only; the fork's deploy callback reads - /// this from the parent). Off by default. + /// `wm-fork/**` branch (needs a credential the server holds; the fork's + /// deploy callback reads this from the parent). Off by default. #[serde(default, skip_serializing_if = "is_false")] pub fork_open_prs: bool, /// Server-owned: the last failure opening a PR for a deploy branch of this @@ -360,6 +361,10 @@ pub struct GitRepositorySettings { /// successful PR; never accepted from clients. #[serde(default, skip_serializing_if = "Option::is_none")] pub open_pr_error: Option, + /// Server-owned: what the repo's credential says about its own expiry and + /// scopes. Written by the credential check, never accepted from clients. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential: Option, } impl GitRepositorySettings { @@ -405,6 +410,44 @@ pub enum AutoPullMode { Polling, } +/// Host whose credential lifecycle Windmill can manage from the repo URL. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum GitCredentialProvider { + Gitlab, +} + +/// What the repo's own credential says about itself, refreshed by asking the +/// host. Server-owned: written by the credential check, never accepted from a +/// client. +/// +/// Absent means the check has not run or the repo carries no credential we can +/// introspect (a GitHub App repo mints tokens per call and has nothing to expire). +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct GitCredentialStatus { + pub provider: GitCredentialProvider, + /// Changes on every rotation, so it identifies the current token, not the + /// credential's whole history. + #[serde(skip_serializing_if = "Option::is_none")] + pub token_id: Option, + /// `None` is a non-expiring token, which only self-managed GitLab can issue + /// (and only for a service account). It means no warning and no rotation. + #[serde(skip_serializing_if = "Option::is_none")] + pub expires_at: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scopes: Vec, + /// Whether *this workspace* renews the credential. That needs a scope which + /// permits it (`api` or `self_rotate`) and a credential this workspace holds: + /// a token carried in the repository URL is the operator's to manage, and one + /// resolved from an ancestor is the ancestor's, so neither is renewed here. + pub rotatable: bool, + /// Unix timestamp (seconds) of the last check. + pub checked_at: i64, + /// Why the last check or rotation failed, cleared by the next success. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + /// Outcome of the most recent auto-pull attempt, surfaced in the UI. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AutoPullStatus { @@ -471,6 +514,11 @@ pub struct AutoPullSettings { pub last_synced_sha: std::collections::HashMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub last_pull_status: Option, + /// Email of the admin this repository's automatic pulls (its own and its forks') + /// apply changes as: whoever last saved the settings with auto pull on. Stamped + /// server-side, never taken from the client. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub enabled_by: Option, } // Manual Debug so the HMAC `webhook_secret` (even encrypted) never lands in logs. @@ -481,6 +529,7 @@ impl std::fmt::Debug for AutoPullSettings { .field("mode", &self.mode) .field("poll_interval_s", &self.poll_interval_s) .field("sync_forks", &self.sync_forks) + .field("enabled_by", &self.enabled_by) .field("webhook_id", &self.webhook_id) .field( "webhook_secret", @@ -516,9 +565,10 @@ impl AutoPullSettings { /// Whether a freshly observed `(git_ref, head_sha)` warrants enqueuing a pull. /// /// A trigger (poll or webhook) is only a hint: we pull when auto-pull is - /// enabled and the observed head differs from the last sha we synced for - /// that ref. Re-observing the same head (e.g. a redundant poll, or the - /// commit our own deploy callback just pushed back) is a no-op. + /// enabled and the observed head differs from the last sha we pulled for + /// that ref. Re-observing the same head (a redundant poll) is a no-op. A + /// commit our own deploy pushed is not: pushes never write here, so the pull + /// it triggers picks up anything pushed under it. pub fn should_pull(&self, git_ref: &str, head_sha: &str) -> bool { self.enabled && self.last_synced_sha.get(git_ref).map(String::as_str) != Some(head_sha) } @@ -820,8 +870,12 @@ pub async fn guest_count_in_window<'c, E: sqlx::Executor<'c, Database = sqlx::Po /// The instance's standing against the guest allowance, as every surface reports it. #[derive(Clone, Debug, Serialize)] pub struct GuestUsage { + /// Whether this deployment can admit guests at all ([`instance_supports_guests`]). + /// Off, every other field is moot and no switch below can turn guests on. + pub available: bool, /// The superadmin switch (`GUEST_ACCESS_DISABLED_SETTING`), which every workspace - /// switch sits under. + /// switch sits under. Reported as stored, so a superadmin sees what they set even + /// where `available` overrules it. pub instance_enabled: bool, /// Distinct guest emails over the trailing `window_days`. pub guest_count: i64, @@ -834,9 +888,33 @@ pub struct GuestUsage { pub guest_seats: i64, } -/// SQL for "the instance admits guests": the superadmin switch, absent meaning on. The -/// setting is read as text before the cast so `true` and `"true"` both count. -fn instance_admits_guests_sql() -> String { +/// What a caller is told when it asks for guests on a deployment that cannot have them. +pub const GUESTS_UNAVAILABLE_MESSAGE: &str = + "Guest access is not available on Windmill Cloud. It requires a self-hosted instance \ + or a dedicated Windmill Cloud deployment."; + +/// Whether guests can exist on this deployment at all. They cannot on the shared cloud: +/// a guest is an identity Windmill itself never vouched for, admitted on the say-so of +/// whoever runs the instance, which is not a call a multi-tenant deployment can make for +/// its tenants. Folded into every guest gate below, so a workspace switch or an app +/// policy left saying `guest` is inert rather than honored. +pub fn instance_supports_guests() -> bool { + !*crate::worker::CLOUD_HOSTED +} + +/// [`instance_supports_guests`] as an error, for the writes that would otherwise store a +/// setting that can never take effect. +pub fn require_guest_support() -> Result<()> { + if instance_supports_guests() { + Ok(()) + } else { + Err(Error::BadRequest(GUESTS_UNAVAILABLE_MESSAGE.to_string())) + } +} + +/// SQL for the superadmin switch alone, absent meaning on. The setting is read as text +/// before the cast so `true` and `"true"` both count. +fn instance_switch_sql() -> String { format!( "NOT COALESCE((SELECT (value #>> '{{}}')::boolean FROM global_settings \ WHERE name = '{}'), false)", @@ -844,9 +922,18 @@ fn instance_admits_guests_sql() -> String { ) } +/// SQL for "the instance admits guests": the superadmin switch, under +/// [`instance_supports_guests`]. +fn instance_admits_guests_sql() -> String { + if !instance_supports_guests() { + return "false".to_string(); + } + instance_switch_sql() +} + pub async fn guest_usage(db: &crate::DB) -> Result { - let instance_admits = instance_admits_guests_sql(); - let instance_enabled: bool = sqlx::query_scalar(&format!("SELECT {instance_admits}")) + let instance_switch = instance_switch_sql(); + let instance_enabled: bool = sqlx::query_scalar(&format!("SELECT {instance_switch}")) .fetch_one(db) .await .map_err(|e| Error::internal_err(format!("reading the instance guest switch: {e:#}")))?; @@ -858,6 +945,7 @@ pub async fn guest_usage(db: &crate::DB) -> Result { 0 }; Ok(GuestUsage { + available: instance_supports_guests(), instance_enabled, guest_count, window_days: GUEST_WINDOW_DAYS, @@ -1700,6 +1788,48 @@ pub async fn workspace_with_fork_ancestors(db: &crate::DB, w_id: &str) -> Result Ok(chain) } +/// The nearest fork ancestor of `w_id` holding a row at `path` in `table`, or `None` when no +/// ancestor does (or `w_id` is not a fork). Fork creation clones trigger and schedule rows down +/// the whole chain, so a row shares its upstream identifier (Kafka group, PG slot, cron) with +/// every ancestor that still has one, not just the direct parent, which may have deleted its +/// copy since. +/// +/// Runs on the caller's connection so it sees the caller's transaction, and uncached: the +/// answer depends on the target table, not only on lineage. +/// +/// `table` is interpolated into SQL, hence `'static`: a trigger's `TABLE_NAME` or a literal, +/// never caller input. Reads lineage for any `w_id` with no authorization check, like +/// [`fork_ancestor_chain`], so the caller must already be authorized for `w_id`. +pub async fn nearest_fork_ancestor_having( + conn: &mut sqlx::PgConnection, + table: &'static str, + w_id: &str, + path: &str, +) -> Result> { + sqlx::query_scalar(&format!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT chain.id FROM chain + JOIN {table} t ON t.workspace_id = chain.id AND t.path = $2 + WHERE chain.depth > 0 + ORDER BY chain.depth LIMIT 1 + "# + )) + .bind(w_id) + .bind(path) + .fetch_optional(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("resolving fork ancestors of {w_id}: {e:#}"))) +} + lazy_static::lazy_static! { /// workspace id -> (root workspace id, expiry ts). Read once per job start, so correctness /// rests on the invalidation rather than on the TTL: every mutation that can change the answer @@ -2627,6 +2757,7 @@ mod tests { webhook_secret: None, webhook_url: None, webhook_error: None, + enabled_by: None, last_synced_sha: synced .iter() .map(|(r, s)| (r.to_string(), s.to_string())) diff --git a/backend/windmill-common/tests/dbt_graph_storage.rs b/backend/windmill-common/tests/dbt_graph_storage.rs index 7b4971312a..3226d27e9e 100644 --- a/backend/windmill-common/tests/dbt_graph_storage.rs +++ b/backend/windmill-common/tests/dbt_graph_storage.rs @@ -7,9 +7,10 @@ use sqlx::{Pool, Postgres}; use windmill_common::dbt_manifest::{ - clear_dbt_editor_graphs, clear_dbt_manifest_version, prune_dbt_run_graphs, - replace_dbt_editor_graph, replace_dbt_manifest, IngestedManifest, IngestedNode, - DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, + clear_dbt_editor_graphs, clear_dbt_manifest_version, clear_dbt_script_state, + clear_dbt_script_state_if_path_retired, move_dbt_script_state, prune_dbt_run_graphs, + replace_dbt_editor_graph, replace_dbt_manifest, IngestedColumnEdge, IngestedManifest, + IngestedNode, DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, }; const WS: &str = "test-workspace"; @@ -52,10 +53,36 @@ fn manifest(names: &[&str]) -> IngestedManifest { .windows(2) .map(|w| (format!("model.p.{}", w[0]), format!("model.p.{}", w[1]))) .collect(), + // Same reason: a project that opted into the analysis pass has these, and + // a fixture without them leaves every column-edge insert and sweep in + // this file unexecuted. + column_edges: names + .windows(2) + .map(|w| IngestedColumnEdge { + parent_unique_id: format!("model.p.{}", w[0]), + parent_column: w[0].to_string(), + child_unique_id: format!("model.p.{}", w[1]), + child_column: w[1].to_string(), + lineage_kind: "copy".to_string(), + }) + .collect(), ..Default::default() } } +/// Column edges of one version, so the sweeps can be shown to reach them. +async fn column_edges_for(db: &Pool, hash: i64) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + WS, + hash + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} + /// Edges for one version, so a test can assert the batched insert ran at all. async fn edges_for(db: &Pool, hash: i64) -> i64 { sqlx::query_scalar!( @@ -135,6 +162,33 @@ async fn an_identical_run_stores_no_snapshot(db: Pool) { assert_eq!(markers(&db, 1).await, 1, "and leaves no marker of its own"); } +/// A column that is projected AND used as a predicate for the same output column +/// has both a `copy` edge and a `scan` one. They are two facts, and the digest +/// counts both — so the uniqueness key has to carry `lineage_kind`, or the +/// second is dropped by `ON CONFLICT DO NOTHING` while the digest still claims +/// it was stored. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn both_kinds_of_one_column_pair_are_stored(db: Pool) { + deploy_script(&db, 1).await; + let pair = |kind: &str| IngestedColumnEdge { + parent_unique_id: "model.p.a".to_string(), + parent_column: "id".to_string(), + child_unique_id: "model.p.b".to_string(), + child_column: "id".to_string(), + lineage_kind: kind.to_string(), + }; + let mut m = manifest(&["a", "b"]); + m.column_edges = vec![pair("copy"), pair("scan")]; + + let mut tx = db.begin().await.unwrap(); + replace_dbt_manifest(&mut tx, WS, PATH, 1, None, &m, "root") + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!(column_edges_for(&db, 1).await, 2, "both kinds survive"); +} + /// A run whose model set differs keeps its own, and the version's is untouched: /// this is what lets an older run page render the project that run built. #[sqlx::test(migrations = "../migrations", fixtures("base"))] @@ -235,7 +289,11 @@ async fn clearing_one_version_leaves_the_others(db: Pool) { // this is where two versions coexist: it pins the batched edge insert // against a real database as well as the version scoping. assert_eq!(edges_for(&db, 1).await, 0, "the cleared version's edges go"); - assert_eq!(edges_for(&db, 2).await, 1, "the other version keeps its own"); + assert_eq!( + edges_for(&db, 2).await, + 1, + "the other version keeps its own" + ); } /// The routes that hard-delete a path clear no graph rows: they delete the @@ -275,6 +333,8 @@ async fn deleting_the_script_cascades_to_every_sidecar(db: Pool) { assert_eq!(nodes_for(&db, 2, DEPLOYED_GRAPH).await, 0); assert_eq!(edges_for(&db, 1).await, 0); assert_eq!(edges_for(&db, 2).await, 0); + assert_eq!(column_edges_for(&db, 1).await, 0); + assert_eq!(column_edges_for(&db, 2).await, 0); assert_eq!(markers_for_path(&db).await, 0); } @@ -315,7 +375,7 @@ async fn the_sweep_takes_old_snapshots_and_spares_the_version(db: Pool tx.commit().await.unwrap(); // Age one snapshot past the window, rows and marker together. - for t in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for t in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "UPDATE {t} SET ingested_at = now() - interval '400 days' WHERE job_id = $1" )) @@ -363,7 +423,11 @@ async fn only_the_newest_deploys_keep_their_graph(db: Pool) { // The newest is always among them: losing the live version's graph would // empty the page of every run of it. assert_eq!(nodes_for(&db, over, DEPLOYED_GRAPH).await, 1); - assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0, "the oldest is reclaimed"); + assert_eq!( + nodes_for(&db, 1, DEPLOYED_GRAPH).await, + 0, + "the oldest is reclaimed" + ); } /// The third provenance: a `parse` of the EDITOR's buffer, which names no @@ -480,17 +544,27 @@ async fn a_version_clear_spares_editor_graphs_and_a_path_clear_does_not(db: Pool replace_dbt_editor_graph(&mut tx, WS, PATH, job, ME, &manifest(&["a"]), "root") .await .unwrap(); - clear_dbt_manifest_version(&mut tx, WS, PATH, 1).await.unwrap(); + clear_dbt_manifest_version(&mut tx, WS, PATH, 1) + .await + .unwrap(); tx.commit().await.unwrap(); assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0); - assert_eq!(editor_nodes(&db, job).await, 1, "the buffer's graph survives"); + assert_eq!( + editor_nodes(&db, job).await, + 1, + "the buffer's graph survives" + ); let mut tx = db.begin().await.unwrap(); clear_dbt_editor_graphs(&mut tx, WS, PATH).await.unwrap(); tx.commit().await.unwrap(); - assert_eq!(editor_nodes(&db, job).await, 0, "retiring the path takes it"); + assert_eq!( + editor_nodes(&db, job).await, + 0, + "retiring the path takes it" + ); } /// A preview names its own PATH and needs only `jobs:run`, so a bound over the @@ -552,3 +626,157 @@ async fn editor_markers(db: &Pool) -> i64 { .unwrap() .unwrap_or(0) } + +/// A deferral resolves a `ref()` through the manifest of the last successful run +/// at this path, so that state has to follow the script the way the retry state +/// does: a rename must not strand it, and a path no live dbt version occupies +/// must not hand its manifest to whatever is created there next. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn environment_state_follows_the_script(db: Pool) { + const MOVED: &str = "f/test/renamed"; + deploy_script(&db, 1).await; + publish_environment_state(&db, PATH).await; + + let mut tx = db.begin().await.unwrap(); + move_dbt_script_state(&mut tx, WS, PATH, MOVED) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, PATH).await, 0); + assert_eq!(environment_states(&db, MOVED).await, 1); + + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state(&mut tx, WS, MOVED).await.unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, MOVED).await, 0); +} + +/// Archiving or deleting ONE version must not take the path's state with it — +/// the live version's next deferral still needs it — while the last one leaving +/// must, or a script later created at that path inherits the previous project's +/// manifest. The condition is a `NOT EXISTS` in raw SQL, so both directions are +/// pinned against a real database. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn state_goes_only_once_no_live_version_is_left(db: Pool) { + deploy_script(&db, 1).await; + deploy_script(&db, 2).await; + publish_environment_state(&db, PATH).await; + + retire(&db, 1).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 1, + "another version is still live here" + ); + + retire(&db, 2).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 0, + "the last one leaving takes it" + ); +} + +async fn retire(db: &Pool, hash: i64) { + sqlx::query!( + "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + WS, + hash + ) + .execute(db) + .await + .unwrap(); +} + +/// The worker publishes under a guard naming the version that ran, and the whole +/// point of it is a job that finishes late: its script can be renamed away and an +/// unrelated one created at the same path while it runs, and that project must +/// not inherit this one's manifest as its deferral state. Enforced in raw SQL, +/// where a refactor can drop a predicate with no type error, so it is pinned +/// against a real database — the same shape `dbt_state::publish` issues. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_late_job_cannot_publish_for_a_path_it_no_longer_owns(db: Pool) { + deploy_script(&db, 1).await; + assert_eq!(guarded_publish(&db, PATH, 1).await, 1, "its own version"); + assert_eq!( + guarded_publish(&db, PATH, 2).await, + 0, + "a version that never lived here" + ); + + // The script is gone from this path and another one takes it. + sqlx::query!( + "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + WS + ) + .execute(&db) + .await + .unwrap(); + deploy_script(&db, 3).await; + assert_eq!( + guarded_publish(&db, PATH, 1).await, + 0, + "the late job's version does not own this path any more" + ); +} + +/// The predicate `dbt_state::publish` locks the script row on, reduced to what it +/// decides. Keep the two in step — this file cannot call `publish` itself, which +/// is `pub(crate)` in `windmill-worker`. +async fn guarded_publish(db: &Pool, path: &str, ran: i64) -> u64 { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text + WHERE EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false + AND language = 'dbt' + AND (hash = $4 OR $4 = ANY(parent_hashes))) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + WS, + path, + uuid::Uuid::from_u128(9), + ran, + ) + .execute(db) + .await + .unwrap() + .rows_affected() +} + +async fn publish_environment_state(db: &Pool, path: &str) { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + WS, + path, + uuid::Uuid::from_u128(9), + ) + .execute(db) + .await + .unwrap(); +} + +async fn environment_states(db: &Pool, path: &str) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} diff --git a/backend/windmill-common/tests/dbt_producer_rules.rs b/backend/windmill-common/tests/dbt_producer_rules.rs new file mode 100644 index 0000000000..f63e0879b5 --- /dev/null +++ b/backend/windmill-common/tests/dbt_producer_rules.rs @@ -0,0 +1,196 @@ +/*! + * One rule, two spellings: "is dbt the sole producer of this warehouse + * relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription + * is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy + * retroactively leaves unwakeable. Both must answer "yes, dormant" only when + * every script writing the relation is a dbt one — a dbt run does not dispatch — + * and "no" both when a native `// materialize manual dbt://…` producer exists and + * when nothing produces the relation yet, which is the ordinary deploy-order + * case. Every way of getting this wrong is silent: a dormant edge on the canvas, + * a refused deploy of a valid pipeline, or a warning that stops appearing. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer}; + +const WS: &str = "test-workspace"; +const RELATION: &str = "main/analytics/orders"; +const SUBSCRIBER: &str = "u/test-user/consumer"; + +async fn plant_producer(db: &Pool, path: &str, language: &str, hash: i64) { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(language) + .execute(db) + .await + .expect("insert script"); + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING", + ) + .bind(WS) + .bind(RELATION) + .bind(path) + .execute(db) + .await + .expect("insert asset"); +} + +async fn plant_subscriber(db: &Pool, path: &str) { + sqlx::query( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, + trigger_ref) + VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)", + ) + .bind(WS) + .bind(path) + .bind(RELATION) + .execute(db) + .await + .expect("insert script_trigger"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn no_producer_is_not_dormant(db: Pool) { + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None, + "a relation nothing produces yet must not refuse the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn dbt_only_producer_is_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None + ); +} + +/// `asset` is keyed by path while `script` holds every version of it, so the +/// language has to be read off the live one: a path converted to dbt still has +/// its old native versions sitting in `script`. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_superseded_native_version_does_not_count(db: Pool) { + plant_producer(&db, "u/test-user/project", "postgresql", 1).await; + sqlx::query("UPDATE script SET archived = true WHERE hash = 1") + .execute(&db) + .await + .expect("archive the old version"); + plant_producer(&db, "u/test-user/project", "dbt", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +/// The rows of the script being deployed describe the version it replaces, so a +/// script dropping its `// materialize` while adding a subscription would +/// otherwise count itself as the producer that wakes it — and commit a dormant +/// edge. It can never be that producer anyway: the dispatcher skips self-loops. +/// Under a rename that write sits at the OLD path, which the deploy is removing +/// in the same uncommitted transaction, so both paths have to be excluded. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_subscriber_is_never_its_own_producer(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_renamed_producer_is_excluded_too(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer( + &db, + WS, + RELATION, + &[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()] + ) + .await + .unwrap(), + Some("u/test-user/project".to_string()), + "the write this deploy is moving off the old path cannot wake the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_agrees_with_the_singular_one(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, "u/test-user/consumer").await; + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → u/test-user/consumer")], + "dbt alone builds it, so the subscription can never be woken" + ); + + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "a native producer wakes it, so the edge is live" + ); +} + +/// The two ways the set form could stop meaning what the singular one means: a +/// relation nothing produces is deploy order rather than a dormant edge, and a +/// subscriber's own write is not a producer that can wake it — the dispatcher +/// skips self-loops, so that edge is dormant and has to be named. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_matches_on_the_edge_cases_too(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, SUBSCRIBER).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "nothing produces it yet, so nothing is dormant" + ); + + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → {SUBSCRIBER}")], + "the subscriber's own write cannot wake it, so dbt is still the sole producer" + ); +} diff --git a/backend/windmill-common/tests/notify_events.rs b/backend/windmill-common/tests/notify_events.rs index 8161130875..285086308e 100644 --- a/backend/windmill-common/tests/notify_events.rs +++ b/backend/windmill-common/tests/notify_events.rs @@ -360,6 +360,77 @@ async fn test_trigger_notify_workspace_key_change(db: Pool) { ); } +/// The address a job runs as is served from a process-local cache, so every change that can move +/// a `(workspace, username)` -> email mapping has to reach the other replicas as an eviction. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_trigger_notify_user_email_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + sqlx::query("UPDATE usr SET email = 'renamed@windmill.dev' WHERE workspace_id = 'test-workspace' AND username = 'test-user'") + .execute(&db) + .await + .expect("Failed to change email"); + + let events = poll_notify_events(&db, before_id) + .await + .expect("Should poll events"); + assert!( + events.iter().any(|e| e.channel == "notify_user_email_change" + && e.payload == "test-workspace:test-user"), + "email change should evict the key it moved" + ); + + // A superadmin outside their workspaces resolves through `password`, which names no + // workspace: the wildcard is the only way to reach that key. `super_admin` is half of what + // that fallback matches on, so losing it moves the mapping just as the address does. + for (label, stmt, expected_aliases) in [ + ( + "email change", + "UPDATE password SET email = 'sa2@windmill.dev' WHERE email = 'test@windmill.dev'", + // old address, new address, and the username that outlives both + vec!["test@windmill.dev", "sa2@windmill.dev", "test-user"], + ), + ( + "demotion", + "UPDATE password SET super_admin = false WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ( + "promotion", + "UPDATE password SET super_admin = true WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ( + "deletion", + "DELETE FROM password WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ] { + let before_id = get_latest_event_id(&db).await.unwrap(); + sqlx::query(stmt) + .execute(&db) + .await + .unwrap_or_else(|e| panic!("Failed to apply superadmin {label}: {e}")); + + let events = poll_notify_events(&db, before_id) + .await + .expect("Should poll events"); + // Every alias the principal can be spelled as, since `resolve_username_to_email` + // matches a `u/` principal against the username or the address. + let evicted: Vec<&str> = events + .iter() + .filter(|e| e.channel == "notify_user_email_change") + .filter_map(|e| e.payload.strip_prefix("*:")) + .collect(); + for alias in expected_aliases { + assert!( + evicted.contains(&alias), + "superadmin {label} should evict {alias}, got {evicted:?}" + ); + } + } +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_trigger_notify_token_invalidation(db: Pool) { // First insert a session token with token_hash and token_prefix diff --git a/backend/windmill-common/tests/permissioned_as_authz.rs b/backend/windmill-common/tests/permissioned_as_authz.rs new file mode 100644 index 0000000000..25c219845f --- /dev/null +++ b/backend/windmill-common/tests/permissioned_as_authz.rs @@ -0,0 +1,81 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::auth::fetch_authed_from_permissioned_as; + +/// The address handed to `fetch_authed_from_permissioned_as` may come from a cache that a +/// username reassignment has outrun. It must not be believed: the workspace role is keyed on the +/// principal while `super_admin` and `email_to_igroup` are keyed on the address, so trusting a +/// stale one would run the new holder's job with the previous holder's instance privileges. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_stale_address_cannot_carry_the_previous_holders_privileges(db: Pool) { + // `test-user` in the fixture is a superadmin with the address `test@windmill.dev`. Free the + // username and hand it to somebody who is not, exactly as an offboard-then-onboard would. + sqlx::query("DELETE FROM usr WHERE workspace_id = 'test-workspace' AND username = 'test-user'") + .execute(&db) + .await + .expect("free the username"); + sqlx::query( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('newcomer@windmill.dev', 'x', 'password', false, true, 'Newcomer')", + ) + .execute(&db) + .await + .expect("create the new account"); + sqlx::query( + "INSERT INTO usr(workspace_id, email, username, is_admin, role) + VALUES ('test-workspace', 'newcomer@windmill.dev', 'test-user', false, 'User')", + ) + .execute(&db) + .await + .expect("reassign the username"); + + // What a replica that has not yet consumed the eviction would pass: the principal is the + // reassigned username, the address is the one it cached for the previous holder. + let authed = fetch_authed_from_permissioned_as( + "u/test-user", + "test@windmill.dev", + "test-workspace", + &db, + ) + .await + .expect("should authenticate the current holder"); + + assert_eq!( + authed.email, "newcomer@windmill.dev", + "the principal's live address must win over the one supplied" + ); + assert!( + !authed.is_admin, + "the new holder must not inherit the previous holder's superadmin" + ); +} + +/// A disabled member still holds its username in the workspace. Workspace usernames are only +/// unique per workspace, so an unrelated instance superadmin can share it, and falling through to +/// the `password` fallback would run the disabled member's jobs as that superadmin. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_disabled_member_never_resolves_to_a_same_named_superadmin(db: Pool) { + sqlx::query( + "UPDATE usr SET disabled = true WHERE workspace_id = 'test-workspace' AND username = 'test-user-2'", + ) + .execute(&db) + .await + .expect("disable the member"); + sqlx::query( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('other-superadmin@windmill.dev', 'x', 'password', true, true, 'Other', 'test-user-2')", + ) + .execute(&db) + .await + .expect("create the same-named superadmin"); + + for supplied in ["test2@windmill.dev", "other-superadmin@windmill.dev"] { + let authed = + fetch_authed_from_permissioned_as("u/test-user-2", supplied, "test-workspace", &db) + .await; + assert!( + authed.is_err(), + "a disabled member must not authenticate (supplied {supplied}): {:?}", + authed.map(|a| (a.email, a.is_admin)) + ); + } +} diff --git a/backend/windmill-common/tests/queue_metrics_series.rs b/backend/windmill-common/tests/queue_metrics_series.rs new file mode 100644 index 0000000000..f6bc7b4056 --- /dev/null +++ b/backend/windmill-common/tests/queue_metrics_series.rs @@ -0,0 +1,136 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::queue_metrics::{read_queue_metrics_series, QUEUE_METRIC_STALE_SECS}; + +const WINDOW: f64 = 3600.0; + +/// Store a sample written `at` seconds after the start of a `WINDOW` ending now. +async fn sample(db: &Pool, id: &str, value: serde_json::Value, at: f64) { + sqlx::query( + "INSERT INTO metrics (id, value, created_at) VALUES ($1, $2, now() - make_interval(secs => $3))", + ) + .bind(id) + .bind(value) + .bind(WINDOW - at) + .execute(db) + .await + .expect("failed to store a metric sample"); +} + +/// The database hands the renderer the last sample before the window, which sets the left edge, +/// and for each slot its peak and its latest value, which the line continues from. +#[sqlx::test(migrations = "../migrations")] +async fn a_series_starts_from_the_sample_before_the_window_and_keeps_each_slot_peak( + db: Pool, +) { + // Before the window: 1, then 2, which is what is in force at the left edge. + sample(&db, "queue_count_t", json!(1), -120.0).await; + sample(&db, "queue_count_t", json!(2), -60.0).await; + // Three samples inside one 30s slot: the line rises to their peak, then drops to the last. + sample(&db, "queue_count_t", json!(5), 605.0).await; + sample(&db, "queue_count_t", json!(9), 612.0).await; + sample(&db, "queue_count_t", json!(4), 620.0).await; + // Drained before the window: nothing left to draw. + sample(&db, "queue_count_gone", json!(3), -300.0).await; + sample(&db, "queue_count_gone", json!(0), -200.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + + assert_eq!( + series.tags.len(), + 1, + "a tag drained before the window is left out" + ); + let tag = &series.tags[0]; + assert_eq!(tag.tag, "t"); + assert!(tag.delay.is_empty()); + + let stale = 620.0 + QUEUE_METRIC_STALE_SECS; + let expected = [ + (0.0, 2.0), + (605.0, 2.0), + (605.0, 9.0), + (620.0, 9.0), + (620.0, 4.0), + (stale, 4.0), + (stale, 0.0), + (WINDOW, 0.0), + ]; + assert_eq!(tag.count.len(), expected.len(), "vertices: {:?}", tag.count); + for ((ms, value), (at, expected_value)) in tag.count.iter().zip(expected) { + let secs = (*ms - series.from) as f64 / 1000.0; + // Samples are stored a few milliseconds before the window is read. + assert!( + (secs - at).abs() < 2.0 && *value == expected_value, + "expected ({at}, {expected_value}), got ({secs}, {value}) in {:?}", + tag.count + ); + } +} + +/// A delay stored as its head's wait start is drawn as that wait, growing a second per second, +/// right up to the zero that closes it. +#[sqlx::test(migrations = "../migrations")] +async fn a_climbing_delay_is_drawn_as_the_wait_of_its_head(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // The head started waiting 30s before the window; heartbeats restate it until the drain. + let head = json!({ "since": now - WINDOW - 30.0 }); + for at in [60.0, 360.0, 660.0] { + sample(&db, "queue_delay_t", head.clone(), at).await; + } + sample(&db, "queue_delay_t", json!(0), 900.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let points = series.tags[0] + .delay + .iter() + .map(|(ms, value)| ((*ms - series.from) as f64 / 1000.0, *value)) + .collect::>(); + + let climb = points + .iter() + .filter(|(_, value)| *value > 0.0) + .collect::>(); + assert!( + climb.len() > 4, + "the climb has vertices along the way: {points:?}" + ); + for (at, value) in &climb { + assert!( + (value - (at + 30.0)).abs() < 2.0, + "off the climb at {at}: {points:?}" + ); + } + let (first, _) = climb[0]; + let (top, _) = climb[climb.len() - 1]; + assert!( + (first - 60.0).abs() < 2.0 && (top - 900.0).abs() < 2.0, + "{points:?}" + ); +} + +/// A climb that drains inside its slot keeps its top, which no stored value holds: it is reached +/// at the next sample. +#[sqlx::test(migrations = "../migrations")] +async fn a_climb_that_drains_inside_its_slot_keeps_its_top(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // All in the 30s slot starting at 600: held at 5s, then climbing from a head queued at 597, + // which is still there when the tag drains at 627, 30s into its wait. + sample(&db, "queue_delay_t", json!(5), 602.0).await; + sample(&db, "queue_delay_t", json!({ "since": now - WINDOW + 597.0 }), 610.0).await; + sample(&db, "queue_delay_t", json!(0), 627.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let top = series.tags[0] + .delay + .iter() + .map(|(_, value)| *value) + .fold(0.0, f64::max); + assert!((top - 30.0).abs() < 2.0, "{:?}", series.tags[0].delay); +} diff --git a/backend/windmill-common/tests/queue_stats.rs b/backend/windmill-common/tests/queue_stats.rs new file mode 100644 index 0000000000..419ad98d56 --- /dev/null +++ b/backend/windmill-common/tests/queue_stats.rs @@ -0,0 +1,69 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::queue::get_queue_stats; + +const WORKSPACE: &str = "test-workspace"; + +async fn queue_job( + db: &Pool, + tag: &str, + priority: Option, + waited_secs: f64, + running: bool, +) { + sqlx::query( + "WITH job AS ( + INSERT INTO v2_job (id, workspace_id, tag) VALUES (gen_random_uuid(), $1, $2) + RETURNING id + ) + INSERT INTO v2_job_queue (id, workspace_id, tag, priority, running, scheduled_for) + SELECT id, $1, $2, $3, $4, now() - make_interval(secs => $5) FROM job", + ) + .bind(WORKSPACE) + .bind(tag) + .bind(priority) + .bind(running) + .bind(waited_secs) + .execute(db) + .await + .expect("failed to queue job"); +} + +/// The delay reported for a tag is that of the job the worker pull takes first, ordered +/// `priority DESC NULLS LAST, scheduled_for`, not simply the oldest one waiting. Running jobs +/// and jobs less than 3 seconds past due are not part of the backlog at all. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn queue_stats_report_the_delay_of_the_job_pulled_next(db: Pool) { + // The oldest job has no priority, so every prioritized job runs before it. + queue_job(&db, "mixed", None, 900.0, false).await; + queue_job(&db, "mixed", Some(1), 600.0, false).await; + queue_job(&db, "mixed", Some(5), 300.0, false).await; + queue_job(&db, "mixed", Some(5), 100.0, false).await; + // Highest priority, but not backlog: already running, or not yet 3 seconds past due. + queue_job(&db, "mixed", Some(9), 1200.0, true).await; + queue_job(&db, "mixed", Some(9), 1.0, false).await; + queue_job(&db, "unprioritized", None, 500.0, false).await; + queue_job(&db, "unprioritized", None, 50.0, false).await; + + let stats = get_queue_stats(&db).await.unwrap(); + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + + let mixed = &stats["mixed"]; + assert_eq!(mixed.count, 4); + assert!( + (mixed.delay - 300.0).abs() < 5.0, + "expected the oldest job of the highest priority, got a delay of {}", + mixed.delay + ); + // The same job's wait start, which the delay is measured from. + assert!((mixed.head_since + mixed.delay - now).abs() < 5.0); + let unprioritized = &stats["unprioritized"]; + assert_eq!(unprioritized.count, 2); + assert!( + (unprioritized.delay - 500.0).abs() < 5.0, + "expected the oldest job, got a delay of {}", + unprioritized.delay + ); +} diff --git a/backend/windmill-common/tests/schema_replica_identity.rs b/backend/windmill-common/tests/schema_replica_identity.rs new file mode 100644 index 0000000000..fb14a5c78e --- /dev/null +++ b/backend/windmill-common/tests/schema_replica_identity.rs @@ -0,0 +1,46 @@ +//! Every table must be replicable. +//! +//! PostgreSQL refuses UPDATE and DELETE on a table that has neither a PRIMARY KEY +//! nor an explicit REPLICA IDENTITY once the database is published to a logical +//! replication slot. That is what a low-downtime major-version upgrade runs on +//! (RDS and Aurora Blue/Green, pglogical) and what every CDC pipeline reads, so a +//! single keyless table blocks the upgrade outright. This runs against a freshly +//! migrated database and fails on the migration that introduces one. + +use sqlx::{Pool, Postgres}; + +/// Partitioned parents are checked alongside ordinary tables: a parent without a +/// key hands the same defect to every partition created under it later. +#[sqlx::test(migrations = "../migrations")] +async fn every_table_is_replicable(db: Pool) -> anyhow::Result<()> { + let offenders: Vec = sqlx::query_scalar( + "SELECT n.nspname || '.' || c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND NOT ( + -- FULL and USING INDEX replicate on their own. + c.relreplident IN ('f', 'i') + -- DEFAULT resolves to the primary key, so it needs one to exist. + -- NOTHING never replicates, primary key or not. + OR (c.relreplident = 'd' AND EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary + )) + ) + ORDER BY 1", + ) + .fetch_all(&db) + .await?; + + assert!( + offenders.is_empty(), + "logical replication will reject UPDATE and DELETE on these tables, because \ + they carry no replica identity it can use: {}. \ + Give each one a primary key -- a natural composite key where every column \ + is NOT NULL, otherwise a surrogate `BIGINT GENERATED ALWAYS AS IDENTITY`.", + offenders.join(", ") + ); + + Ok(()) +} diff --git a/backend/windmill-common/tests/usr_accepts_email.rs b/backend/windmill-common/tests/usr_accepts_email.rs new file mode 100644 index 0000000000..624d8812b4 --- /dev/null +++ b/backend/windmill-common/tests/usr_accepts_email.rs @@ -0,0 +1,59 @@ +//! `usr_accepts_email` predicts whether `usr` will store an address by evaluating +//! `PROPER_EMAIL_PATTERN` in the database. It only stays right while that text matches the +//! `proper_email` constraint and the width matches the column: each sample below must be +//! stored by `usr` exactly when the check accepts it, and everything `VALID_EMAIL` accepts +//! within the width must be stored too. + +use sqlx::{Pool, Postgres}; +use windmill_common::users::{usr_accepts_email, EMAIL_COLUMN_MAX_LEN, VALID_EMAIL}; + +#[sqlx::test(migrations = "../migrations")] +async fn usr_accepts_email_agrees_with_the_constraint(db: Pool) -> anyhow::Result<()> { + let domain = "@example.com"; + let widest = format!( + "{}{domain}", + "a".repeat(EMAIL_COLUMN_MAX_LEN - domain.len()) + ); + let too_wide = format!("a{widest}"); + for email in [ + "alice@example.com", + "Alice@Example.COM", + "alice.bob+tag@sub.example.co.uk", + "\"quoted\"@example.com", + "\"quoted local\"@example.com", + "alice@[192.168.0.1]", + widest.as_str(), + too_wide.as_str(), + "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed", + // Unicode case folding would map the long s and the Kelvin sign into `[a-z]`. + "u\u{17f}er@example.com", + "alice@example\u{212a}.com", + "alice", + "alice@example", + "alice@@example.com", + "alice @example.com", + "alice@example.com\nbob@example.com", + "", + ] { + let mut tx = db.begin().await?; + let stored = sqlx::query( + "INSERT INTO usr (workspace_id, username, email, is_admin, operator) + VALUES ('admins', 'probe', $1, false, false)", + ) + .bind(email) + .execute(&mut *tx) + .await + .is_ok(); + tx.rollback().await?; + + assert_eq!( + stored, + usr_accepts_email(&db, email).await?, + "{email:?}: `usr` and usr_accepts_email disagree" + ); + if VALID_EMAIL.is_match(email) && email.len() <= EMAIL_COLUMN_MAX_LEN { + assert!(stored, "{email:?}: VALID_EMAIL accepts what `usr` rejects"); + } + } + Ok(()) +} diff --git a/backend/windmill-dep-map/src/ci_tests.rs b/backend/windmill-dep-map/src/ci_tests.rs index 79f1b7e30b..8f56111b35 100644 --- a/backend/windmill-dep-map/src/ci_tests.rs +++ b/backend/windmill-dep-map/src/ci_tests.rs @@ -19,3 +19,13 @@ pub async fn trigger_ci_tests_for_item( ) -> error::Result> { Ok(vec![]) } + +#[cfg(not(feature = "private"))] +pub async fn trigger_all_ci_tests( + _db: &sqlx::Pool, + _w_id: &str, + _email: &str, + _username: &str, +) -> error::Result> { + Ok(vec![]) +} diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index 148746dcca..2b691ee48e 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,8 +9,8 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] -private = ["windmill-common/private"] -enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +private = ["windmill-common/private", "windmill-dep-map/private", "windmill-audit/private"] +enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise", "windmill-audit/enterprise"] all_sqlx_features = ["enterprise"] default = [] @@ -22,5 +22,7 @@ serde_json.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true +windmill-dep-map.workspace = true +windmill-audit.workspace = true regex = "1.10.3" tokio = { workspace = true, features = ["full"] } \ No newline at end of file diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 2bfc7d2f77..d65e230988 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -14,12 +14,21 @@ pub mod git_sync_oss; #[cfg(feature = "private")] pub use git_sync_ee::{ - enqueue_git_pull_dry_run, enqueue_git_pull_job, handle_deployment_metadata, - handle_deployment_metadata_batch, handle_fork_branch_creation, persist_auto_pull_state, - reconcile_and_enqueue_pull, reconcile_fork_branch_pull, record_auto_pull_failure, + clear_auto_pull_failure, enqueue_git_pull_dry_run, enqueue_git_pull_job, + handle_deployment_metadata, handle_deployment_metadata_batch, handle_fork_branch_creation, + persist_auto_pull_state, reconcile_and_enqueue_pull, reconcile_fork_branch_pull, + record_auto_pull_failure, record_synced_head, sweep_ci_test_checks, tally_deployed_object_changes, }; +// The CI-test check exists only on enterprise builds; `private` alone (the CE image) +// compiles git_sync_ee without them. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub use git_sync_ee::{ + ensure_ci_test_check_for_pr, evaluate_and_conclude_ci_test_checks, + post_ci_test_check_not_applicable, resolve_pr_head_workspace, +}; + #[cfg(not(feature = "private"))] pub use git_sync_oss::{ handle_deployment_metadata, handle_deployment_metadata_batch, handle_fork_branch_creation, diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index cc6c61be81..27a50ba07a 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -1,8 +1,8 @@ use crate::{ classify_read_failure, decrypt_oauth_data, delete_native_trigger, delete_token_by_hash, get_native_trigger, list_native_triggers, lock::TriggerLock, map_external_error, - map_external_error_with, rotate_webhook_token, store_native_trigger, - sync::EXTERNAL_TRIGGER_MISSING_ERROR, update_native_trigger_error, + map_external_error_with, rotate_webhook_token, set_native_trigger_enabled, + store_native_trigger, sync::EXTERNAL_TRIGGER_MISSING_ERROR, update_native_trigger_error, update_native_trigger_if_runnable_unchanged, webhook_token_label, webhook_token_scopes, External, ExternalReadFailure, NativeTrigger, NativeTriggerConfig, NativeTriggerData, ServiceName, @@ -239,6 +239,7 @@ async fn create_native_trigger( &config, service_config, data.summary.as_deref(), + data.enabled, ) .await?; @@ -603,6 +604,87 @@ async fn delete_native_trigger_handler( Ok(format!("Native trigger deleted")) } +#[derive(Debug, Deserialize)] +pub struct SetEnabledPayload { + pub enabled: bool, +} + +/// Pause or resume a trigger, without touching its registration on the external service. +/// +/// Leaving the webhook registered is what makes this reversible: services drop or deactivate a +/// subscription that keeps failing, so a paused trigger keeps answering deliveries normally and +/// simply starts no job. +async fn set_native_trigger_enabled_handler( + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, external_id)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) + .await? + .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; + + check_scopes(&authed, || { + format!("native_triggers:write:{}", &existing.script_path) + })?; + require_is_writer_on_runnable( + &authed, + &existing.script_path, + existing.is_flow, + &workspace_id, + db.clone(), + ) + .await?; + + let updated = set_native_trigger_enabled( + &mut *tx, + &workspace_id, + service_name, + &external_id, + payload.enabled, + ) + .await?; + + // The read above takes no row lock, so a concurrent delete can land in between; reporting + // success then would tell the caller a trigger that is gone had been paused. + if !updated { + return Err(Error::NotFound(format!( + "Native trigger not found: {}", + external_id + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!( + "native_triggers.{}.{}", + service_name, + if payload.enabled { "enable" } else { "disable" } + ), + ActionKind::Update, + &workspace_id, + Some(&external_id), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!( + "Native trigger {}", + if payload.enabled { + "enabled" + } else { + "disabled" + } + )) +} + async fn list_native_triggers_handler( Extension(service_name): Extension, authed: ApiAuthed, @@ -642,6 +724,10 @@ pub fn service_routes(handler: T) -> Router { .route( "/delete/{external_id}", delete(delete_native_trigger_handler::), + ) + .route( + "/setenabled/{external_id}", + post(set_native_trigger_enabled_handler::), ); standard_routes diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 4400091ed6..570319ee0e 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -226,6 +226,10 @@ pub struct NativeTrigger { pub created_at: DateTime, pub updated_at: DateTime, pub summary: Option, + /// Whether incoming webhooks for this trigger start a job. Operational state: a create sets + /// its initial value and `setenabled` is its only mutator afterwards, so saving a + /// configuration can never silently re-enable a trigger someone paused. + pub enabled: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -235,12 +239,20 @@ pub struct NativeTriggerConfig { pub webhook_token: String, } +fn default_true() -> bool { + true +} + #[derive(Debug, Serialize, Deserialize)] pub struct NativeTriggerData { pub script_path: String, pub is_flow: bool, pub service_config: C, pub summary: Option, + /// Honoured on create only, so a trigger can be registered already paused in one request. + /// An update ignores it: `setenabled` is the only way to change an existing trigger's state. + #[serde(default = "default_true")] + pub enabled: bool, } #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] @@ -1179,11 +1191,15 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> config: &NativeTriggerConfig, service_config: C, summary: Option<&str>, + enabled: bool, ) -> Result<()> { use windmill_common::auth::hash_token; let webhook_token_hash = hash_token(&config.webhook_token); + // `enabled` is set by the INSERT alone: writing it here rather than in a follow-up statement + // is what keeps a trigger created paused from ever being visible, and therefore runnable, in + // any other state. The conflict branch leaves it untouched for the mirror-image reason. sqlx::query!( r#" INSERT INTO native_trigger ( @@ -1194,9 +1210,10 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> is_flow, webhook_token_hash, service_config, - summary + summary, + enabled ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8 + $1, $2, $3, $4, $5, $6, $7, $8, $9 ) ON CONFLICT (external_id, workspace_id, service_name) DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW() @@ -1209,6 +1226,7 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> webhook_token_hash, sqlx::types::Json(service_config) as _, summary, + enabled, ) .execute(db) .await?; @@ -1405,7 +1423,8 @@ pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( error, created_at, updated_at, - summary + summary, + enabled FROM native_trigger WHERE @@ -1444,7 +1463,8 @@ pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = P error, created_at, updated_at, - summary + summary, + enabled FROM native_trigger WHERE @@ -1491,7 +1511,8 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> nt.error, nt.created_at, nt.updated_at, - nt.summary + nt.summary, + nt.enabled FROM native_trigger nt WHERE @@ -1555,6 +1576,71 @@ pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Po Ok(()) } +/// Pause or resume a trigger. Returns `false` when there is no such trigger. +/// +/// Callers MUST have verified write access to the trigger's runnable: this writes operational +/// state and performs no authorization of its own. +pub async fn set_native_trigger_enabled<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + enabled: bool, +) -> Result { + // `updated_at` is the row version `record_reregistration` conditions on, so leave it alone: + // pausing a trigger must not make a registration that is mid-flight discard its result. + let updated = sqlx::query!( + r#" + UPDATE native_trigger + SET enabled = $1 + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + enabled, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await? + .rows_affected(); + + Ok(updated > 0) +} + +/// Whether a webhook arriving for this trigger should start a job. +/// +/// A trigger Windmill no longer knows about counts as enabled: the token in the URL is what +/// authorizes the run, and this is a pause switch, not a second authorization check. It reads +/// nothing a caller could not already learn from the trigger it is delivering for, so it needs no +/// authorization of its own — but it also grants none, and must not be used as one. +pub async fn native_trigger_is_enabled<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result { + let enabled = sqlx::query_scalar!( + r#" + SELECT enabled + FROM native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id, + ) + .fetch_optional(db) + .await?; + + Ok(enabled.unwrap_or(true)) +} + pub async fn update_native_trigger_service_config< 'c, E: sqlx::Executor<'c, Database = Postgres>, @@ -1701,6 +1787,11 @@ pub async fn delete_workspace_integration( /// /// `external_id` is optional because during CREATE we don't have it yet /// (it's returned by the external service). During UPDATE, we have it. +/// +/// Every registered URL MUST end up carrying it: it is the only thing a delivery identifies its +/// trigger by, so a service that leaves it out ships a disable switch that silently does nothing. +/// A handler that returns `None` from `service_config_from_create_response` gets this for free — +/// `create_native_trigger` then runs the `update` cycle that re-registers with the assigned id. pub fn generate_webhook_service_url( base_url: &str, w_id: &str, diff --git a/backend/windmill-native-triggers/src/rename.rs b/backend/windmill-native-triggers/src/rename.rs index a7168f5c7e..caae99028c 100644 --- a/backend/windmill-native-triggers/src/rename.rs +++ b/backend/windmill-native-triggers/src/rename.rs @@ -135,6 +135,7 @@ async fn reregister_one( is_flow: trigger.is_flow, service_config, summary: trigger.summary.clone(), + enabled: trigger.enabled, }; // The token is scoped to the runnable path and only its hash is kept, so pointing the webhook diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index 32fde93b81..a8a8e83883 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -88,8 +88,10 @@ pub struct OAuthConfig { #[serde(default = "empty_string")] pub token_url: String, pub userinfo_url: Option, - /// The registry JSON may also carry `scope_options`, a frontend-only pick - /// list for the connect dialog; it is deliberately not modelled here. + /// The registry JSON may also carry two frontend-only keys for the connect + /// dialog, deliberately not modelled here: `scope_options`, a scope pick + /// list, and `resource_fields`, the fields of the resource type the dialog + /// asks for once the token is in (Snowflake's database and warehouse). pub scopes: Option>, /// Default scopes for the client-credentials (2-legged) flow. These differ /// from the authorization-code `scopes` for most providers (member/consent diff --git a/backend/windmill-object-store/Cargo.toml b/backend/windmill-object-store/Cargo.toml index 3602007d8c..90af4f7e7c 100644 --- a/backend/windmill-object-store/Cargo.toml +++ b/backend/windmill-object-store/Cargo.toml @@ -58,3 +58,4 @@ aws-credential-types = { workspace = true, optional = true } tempfile.workspace = true tokio = { workspace = true, features = ["rt", "macros"] } object_store.workspace = true +serial_test = "3" diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index cc588d485f..edc3ea9149 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -68,8 +68,8 @@ pub mod object_store_reexports { pub use object_store::path::Path; pub use object_store::{ Attribute, Attributes, Error as ObjectStoreError, GetOptions, GetRange, GetResult, - ObjectStore, PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult, - WriteMultipart, + ObjectMeta, ObjectStore, PutMode, PutMultipartOpts, PutOptions, PutPayload, PutResult, + Result as ObjectStoreResult, UpdateVersion, WriteMultipart, }; } @@ -118,6 +118,10 @@ pub fn object_store_error_to_error(err: object_store::Error) -> error::Error { pub struct ExpirableObjectStore { pub store: Arc, pub refresh: Option, + /// What locates the store's objects ([`object_store_location`]), for a store built from + /// settings. Kept with the store rather than read off the settings again, so a server + /// whose reload is still pending never names one store by another's location. + pub location: Option, } #[cfg(feature = "parquet")] @@ -155,44 +159,234 @@ impl ObjectStoreRefresh { #[cfg(feature = "parquet")] impl From> for ExpirableObjectStore { fn from(store: Arc) -> Self { - Self { store, refresh: None } + Self { store, refresh: None, location: None } } } #[cfg(feature = "parquet")] lazy_static::lazy_static! { pub static ref OBJECT_STORE_SETTINGS: Arc>> = Arc::new(RwLock::new(None)); + + /// Worker-group override of the store backing the *dependency cache* only: venvs, language + /// bundles and compiled binaries, which a worker both writes and reads back itself. + /// Everything the server also reads — job results, logs, codebases, app assets — stays on + /// [`OBJECT_STORE_SETTINGS`], which a worker-local redirect would make unreachable. + static ref CACHE_OBJECT_STORE_OVERRIDE: Arc>> = Arc::new(RwLock::new(None)); + + /// The config [`CACHE_OBJECT_STORE_OVERRIDE`] was built from, so a rebuild that fails for a + /// config already being served can keep serving it. Locked after the store, never before. + static ref CACHE_OVERRIDE_APPLIED: Arc>> = Arc::new(RwLock::new(None)); + + /// Held across a whole [`reload_cache_object_store_override`], build included, so that the + /// override's flag, store and applied config only ever move together. + static ref CACHE_OVERRIDE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(()); +} + +/// Whether a worker-group cache override is configured, held apart from the store it built so +/// that a configured-but-broken override reads as "no cache store" instead of silently falling +/// back to the instance bucket the operator redirected away from. +#[cfg(feature = "parquet")] +static CACHE_OBJECT_STORE_OVERRIDDEN: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Bumped by every [`reload_cache_object_store_override`] at entry. Builds are slow and several +/// callers race — a config change, the retry behind a failed one, a license-plan change — and the +/// lock alone would only order them by arrival, so a reload that lost its claim while waiting +/// drops out rather than installing a store the group has already moved off. +#[cfg(feature = "parquet")] +static CACHE_OVERRIDE_GENERATION: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +#[cfg(feature = "parquet")] +async fn resolve_object_store( + settings_lock: &RwLock>, +) -> Option> { + resolve_object_store_with_location(settings_lock) + .await + .map(|(store, _)| store) +} + +#[cfg(feature = "parquet")] +async fn resolve_object_store_with_location( + settings_lock: &RwLock>, +) -> Option<(Arc, Option)> { + let settings = settings_lock.read().await; + let Some(s) = settings.as_ref() else { + return None; + }; + match &s.refresh { + Some(refresh) if refresh.refresh_needed() => { + let refresh = refresh.clone(); + let refreshed_from = s.store.clone(); + drop(settings); + let new_store = refresh.refresh().await?; + let mut settings = settings_lock.write().await; + match settings.as_ref() { + // A reload may have installed a different store while the credentials were + // being minted; that one reflects newer config, so the refresh is stale. + Some(current) if !Arc::ptr_eq(¤t.store, &refreshed_from) => { + Some((current.store.clone(), current.location.clone())) + } + Some(_) => { + let found = (new_store.store.clone(), new_store.location.clone()); + *settings = Some(new_store); + Some(found) + } + // Cleared while refreshing. + None => None, + } + } + _ => Some((s.store.clone(), s.location.clone())), + } } #[cfg(feature = "parquet")] pub async fn get_object_store() -> Option> { - let settings = OBJECT_STORE_SETTINGS.read().await; - if let Some(s) = settings.as_ref() { - match &s.refresh { - Some(refresh) => { - if refresh.refresh_needed() { - let refresh = refresh.clone(); - drop(settings); - let new_store = refresh.refresh().await; - if let Some(new_store) = new_store { - let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await; - let arc = new_store.store.clone(); - *s3_cache_settings = Some(new_store); - return Some(arc); - } else { - return None; - } - } else { - return Some(s.store.clone()); - } - } - None => { - return Some(s.store.clone()); - } - } - } else { - return None; + resolve_object_store(&OBJECT_STORE_SETTINGS).await +} + +/// The instance object store with what locates its objects ([`object_store_location`]), +/// read together; the location is `None` for a store installed without settings. +#[cfg(feature = "parquet")] +pub async fn get_object_store_with_location() -> Option<(Arc, Option)> { + resolve_object_store_with_location(&OBJECT_STORE_SETTINGS).await +} + +/// The store the dependency cache reads and writes: the worker group's override when it has one, +/// the instance object store otherwise. Anything the server must also reach goes through +/// [`get_object_store`] instead. +#[cfg(feature = "parquet")] +pub async fn get_cache_object_store() -> Option> { + if CACHE_OBJECT_STORE_OVERRIDDEN.load(std::sync::atomic::Ordering::Relaxed) { + return resolve_object_store(&CACHE_OBJECT_STORE_OVERRIDE).await; } + resolve_object_store(&OBJECT_STORE_SETTINGS).await +} + +/// True when an override is configured but has no usable store. The caller's short retry is the +/// fast path back; this is the backstop, and the full settings reload it rides on is 12h apart by +/// default (`SETTINGS_RELOAD_PERIOD_SECS`), so an outage outlasting the retry keeps the group's +/// dependency cache local until then or until someone edits the group config. +#[cfg(feature = "parquet")] +pub async fn cache_object_store_override_failed() -> bool { + CACHE_OBJECT_STORE_OVERRIDDEN.load(std::sync::atomic::Ordering::Relaxed) + && CACHE_OBJECT_STORE_OVERRIDE.read().await.is_none() +} + +/// Apply the `object_store_cache_config` of this worker's group. `None` (or JSON null) drops the +/// override and returns the worker to the instance object store. +/// +/// Returns [`ObjectStoreReload::Later`] when the store did not build for a reason that may pass — +/// the caller is expected to retry shortly, as `initial_load` does for the instance store. +#[cfg(feature = "parquet")] +pub async fn reload_cache_object_store_override( + db: &windmill_common::DB, + settings: Option, +) -> ObjectStoreReload { + use std::sync::atomic::Ordering; + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + // Claim a generation, then take the lock: every state transition below happens inside one + // critical section, and a caller that lost its claim while waiting drops out rather than + // installing what the group has already moved off. + let generation = CACHE_OVERRIDE_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; + let _transition = CACHE_OVERRIDE_LOCK.lock().await; + if CACHE_OVERRIDE_GENERATION.load(Ordering::SeqCst) != generation { + return ObjectStoreReload::Never; + } + + // DISABLE_S3_STORE turns off the instance object store for this process; a group override + // must not be a way back in. + let store_disabled = std::env::var("DISABLE_S3_STORE") + .ok() + .is_some_and(|x| x == "1" || x == "true"); + + let Some(settings) = settings.filter(|v| !v.is_null() && !store_disabled) else { + if CACHE_OBJECT_STORE_OVERRIDDEN.swap(false, Ordering::Relaxed) { + clear_cache_object_store_override().await; + tracing::info!( + "Worker group object store cache override removed, falling back to the instance object store" + ); + } + return ObjectStoreReload::Never; + }; + + // Enterprise-only, so anything else — Community, including a CE build reaching this through + // the config-as-code API, and Pro — must not get a store, and a plan that stops being + // Enterprise must drop one loaded while it still was. + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + tracing::error!( + "Object store cache override requires an enterprise license, ignoring it for this worker group" + ); + if CACHE_OBJECT_STORE_OVERRIDDEN.swap(false, Ordering::Relaxed) { + clear_cache_object_store_override().await; + } + return ObjectStoreReload::Never; + } + + apply_cache_object_store_override(db, settings).await +} + +/// The half of [`reload_cache_object_store_override`] past the entitlement gate: build the store +/// and commit it. Split out so the commit rules are testable without a license plan. +#[cfg(feature = "parquet")] +async fn apply_cache_object_store_override( + db: &windmill_common::DB, + settings: serde_json::Value, +) -> ObjectStoreReload { + use std::sync::atomic::Ordering; + + // Claim the override before building it: until a store is in place the dependency cache + // must stay local-only rather than reach for the instance bucket. + CACHE_OBJECT_STORE_OVERRIDDEN.store(true, Ordering::Relaxed); + + let (store, reload) = match serde_json::from_value::(settings.clone()) { + Ok(setting) => match build_object_store_from_settings(setting, Some(db)).await { + Ok(store) => (Some(store), ObjectStoreReload::Never), + Err(e) => { + tracing::error!( + "Error building the worker group object store cache override, the dependency cache stays local to this worker until it builds: {e:?}" + ); + (None, ObjectStoreReload::Later) + } + }, + // A malformed config will read the same on every retry. + Err(e) => { + tracing::error!( + "Error parsing the worker group object store cache override, the dependency cache stays local to this worker: {e:?}" + ); + (None, ObjectStoreReload::Never) + } + }; + + let mut current = CACHE_OBJECT_STORE_OVERRIDE.write().await; + match store { + Some(store) => { + *current = Some(store); + *CACHE_OVERRIDE_APPLIED.write().await = Some(settings); + tracing::info!( + "Dependency cache of this worker group now uses its own object store, not the instance one" + ); + } + // A rebuild that failed for the config already being served leaves that store in place: + // the group is entitled to it, and dropping it would take the whole group's cache local + // over a transient error. A *different* config failing must still clear, or the worker + // would keep writing to the bucket the operator redirected it away from. + None if current.is_some() + && CACHE_OVERRIDE_APPLIED.read().await.as_ref() == Some(&settings) => {} + None => { + *current = None; + *CACHE_OVERRIDE_APPLIED.write().await = None; + } + } + reload +} + +/// Drop the override store and the config it was built from, in that lock order. +#[cfg(feature = "parquet")] +async fn clear_cache_object_store_override() { + *CACHE_OBJECT_STORE_OVERRIDE.write().await = None; + *CACHE_OVERRIDE_APPLIED.write().await = None; } #[cfg(feature = "parquet")] @@ -248,20 +442,22 @@ pub async fn reload_object_store_setting(db: &windmill_common::DB) -> ObjectStor tracing::error!("S3 cache is not available for pro plan"); return ObjectStoreReload::Never; } - *s3_cache_settings = build_s3_client_from_settings(S3Settings { - bucket: None, - region: None, - access_key: None, - secret_key: None, - endpoint: None, - store_logs: None, - path_style: None, - allow_http: None, - port: None, - }) + *s3_cache_settings = build_object_store_from_settings( + ObjectSettings::S3(S3Settings { + bucket: None, + region: None, + access_key: None, + secret_key: None, + endpoint: None, + store_logs: None, + path_style: None, + allow_http: None, + port: None, + }), + Some(db), + ) .await .ok() - .map(|x| ExpirableObjectStore::from(x)) } else { *s3_cache_settings = None; } @@ -713,19 +909,49 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes { } } +/// What locates a store's objects: endpoint, port, region and bucket (or account and +/// container, or root), never the credentials, which rotate. Two stores with the same +/// location hold the same objects. +pub fn object_store_location(resource: &ObjectStoreResource) -> String { + match resource { + ObjectStoreResource::S3(s) => format!( + "s3:{}:{}:{}:{}", + s.endpoint, + s.port.unwrap_or_default(), + s.region, + s.bucket + ), + ObjectStoreResource::Azure(a) => format!( + "azure:{}:{}:{}", + a.endpoint.as_deref().unwrap_or_default(), + a.account_name, + a.container_name + ), + ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), + ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), + } +} + #[cfg(feature = "parquet")] pub async fn build_object_store_from_settings( settings: ObjectSettings, init_private_key: Option<&windmill_common::DB>, ) -> error::Result { + let located = + |store: Arc, resource: ObjectStoreResource| ExpirableObjectStore { + store, + refresh: None, + location: Some(object_store_location(&resource)), + }; match settings { - ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings) - .await - .map(|x| ExpirableObjectStore::from(x)), - ObjectSettings::Azure(azure_settings) => { - let azure_blob_resource = azure_settings; - build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x)) + ObjectSettings::S3(s3_settings) => { + let s3_resource = s3_resource_from_settings(s3_settings); + build_s3_client(&s3_resource) + .await + .map(|x| located(x, ObjectStoreResource::S3(s3_resource))) } + ObjectSettings::Azure(azure_settings) => build_azure_blob_client(&azure_settings) + .map(|x| located(x, ObjectStoreResource::Azure(azure_settings))), ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => { let token_generator = crate::job_s3_helpers_oss::TokenGenerator::AsServerInstance(); let res = crate::job_s3_helpers_oss::generate_s3_aws_oidc_resource( @@ -740,17 +966,14 @@ pub async fn build_object_store_from_settings( .map(|x| ExpirableObjectStore { store: x, refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())), + location: Some(object_store_location(&res)), }) } - ObjectSettings::Gcs(gcs_settings) => { - let gcs_resource = gcs_settings; - build_gcs_client(&gcs_resource) - .await - .map(|x| ExpirableObjectStore::from(x)) - } - ObjectSettings::Filesystem(fs) => { - build_filesystem_client(&fs.root_path).map(|x| ExpirableObjectStore::from(x)) - } + ObjectSettings::Gcs(gcs_settings) => build_gcs_client(&gcs_settings) + .await + .map(|x| located(x, ObjectStoreResource::Gcs(gcs_settings))), + ObjectSettings::Filesystem(fs) => build_filesystem_client(&fs.root_path) + .map(|x| located(x, ObjectStoreResource::Filesystem(fs))), } } @@ -763,14 +986,14 @@ fn none_if_empty(s: Option) -> Option { } } +/// The S3 resource instance settings resolve to, the environment filling in what they +/// leave out. #[cfg(feature = "parquet")] -pub async fn build_s3_client_from_settings( - settings: S3Settings, -) -> error::Result> { +fn s3_resource_from_settings(settings: S3Settings) -> S3Resource { let region = none_if_empty(settings.region) .unwrap_or_else(|| std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string())); - let s3_resource = S3Resource { + S3Resource { endpoint: none_if_empty(settings.endpoint).unwrap_or_else(|| { std::env::var("S3_ENDPOINT").unwrap_or_else(|_| format!("s3.{region}.amazonaws.com")) }), @@ -785,9 +1008,7 @@ pub async fn build_s3_client_from_settings( port: settings.port, token: None, expiration: None, - }; - - build_s3_client(&s3_resource).await + } } // Resolving the default chain goes over the network (ECS/IMDS) on instances relying on an @@ -2361,10 +2582,130 @@ mod tests { .contains("Error building filesystem object store")); } + /// A worker group override that is configured but has no usable store must leave the + /// dependency cache with no object store at all. Falling back to the instance one would + /// write the group's cache into the bucket the operator redirected it away from. + // Serialized with the other test that swaps OBJECT_STORE_SETTINGS: the store is + // process-global and CI runs this binary with --test-threads=10. + #[cfg(feature = "parquet")] + #[tokio::test] + #[serial_test::serial(object_store_settings)] + async fn test_get_cache_object_store_override() { + use object_store::{path::Path, ObjectStore, PutPayload}; + use std::sync::atomic::Ordering; + + async fn marker_of(store: &Arc) -> String { + let bytes = store.get(&Path::from("marker")).await.unwrap(); + String::from_utf8(bytes.bytes().await.unwrap().to_vec()).unwrap() + } + + let instance_dir = tempfile::tempdir().unwrap(); + let instance = build_filesystem_client(instance_dir.path().to_str().unwrap()).unwrap(); + instance + .put(&Path::from("marker"), PutPayload::from("instance")) + .await + .unwrap(); + + let group_dir = tempfile::tempdir().unwrap(); + let group = build_filesystem_client(group_dir.path().to_str().unwrap()).unwrap(); + group + .put(&Path::from("marker"), PutPayload::from("group")) + .await + .unwrap(); + + *OBJECT_STORE_SETTINGS.write().await = Some(ExpirableObjectStore::from(instance)); + + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "instance"); + assert!(!cache_object_store_override_failed().await); + + CACHE_OBJECT_STORE_OVERRIDDEN.store(true, Ordering::Relaxed); + *CACHE_OBJECT_STORE_OVERRIDE.write().await = Some(ExpirableObjectStore::from(group)); + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "group"); + + // Configured but unbuilt, as a failed reload leaves it: no store at all, rather than the + // instance bucket the operator redirected the group away from. + *CACHE_OBJECT_STORE_OVERRIDE.write().await = None; + assert!(get_cache_object_store().await.is_none()); + assert!(cache_object_store_override_failed().await); + + // The teardown branch returns before the pool is used, so a lazy one is enough. + let db = sqlx::postgres::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); + reload_cache_object_store_override(&db, None).await; + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "instance"); + assert!(!cache_object_store_override_failed().await); + + *OBJECT_STORE_SETTINGS.write().await = None; + } + + /// A rebuild is triggered by any edit to the group config, not only by editing the store, so + /// a build that fails for the config already installed must leave it alone — otherwise a + /// renamed worker tag plus one flaky token mint takes the whole group's cache local. A + /// *different* config failing still has to clear it. + #[cfg(feature = "parquet")] + #[tokio::test] + #[serial_test::serial(object_store_settings)] + async fn test_failed_rebuild_keeps_the_store_serving_the_same_config() { + let db = sqlx::postgres::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); + let dir = tempfile::tempdir().unwrap(); + let settings = serde_json::json!({ + "type": "Filesystem", "root_path": dir.path().to_str().unwrap() + }); + + apply_cache_object_store_override(&db, settings.clone()).await; + assert!(get_cache_object_store().await.is_some()); + + // Same config, now unbuildable: the store it already produced stays. + dir.close().unwrap(); + apply_cache_object_store_override(&db, settings).await; + assert!(get_cache_object_store().await.is_some()); + + // A different config that will not build must not leave the old bucket in place. + let moved = serde_json::json!({ "type": "Filesystem", "root_path": "/proc/nonexistent" }); + apply_cache_object_store_override(&db, moved).await; + assert!(get_cache_object_store().await.is_none()); + assert!(cache_object_store_override_failed().await); + + reload_cache_object_store_override(&db, None).await; + } + + /// A store built from settings is located by where its objects are, not by how the + /// client describes itself: an S3 client prints only its bucket, so the same bucket name + /// on another endpoint would otherwise pass for the same store. + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_settings_store_location_tells_endpoints_apart() { + let s3 = |endpoint: &str| { + ObjectSettings::S3(S3Settings { + bucket: Some("windmill".to_string()), + region: Some("us-east-1".to_string()), + access_key: Some("key".to_string()), + secret_key: Some("secret".to_string()), + endpoint: Some(endpoint.to_string()), + allow_http: Some(true), + path_style: Some(true), + store_logs: None, + port: None, + }) + }; + let a = build_object_store_from_settings(s3("minio.internal:9000"), None) + .await + .unwrap(); + let b = build_object_store_from_settings(s3("s3.us-east-1.amazonaws.com"), None) + .await + .unwrap(); + assert_eq!(a.store.to_string(), b.store.to_string()); + assert!(a.location.is_some()); + assert_ne!(a.location, b.location); + } + // --- get_logs_from_store test --- #[cfg(feature = "parquet")] #[tokio::test] + #[serial_test::serial(object_store_settings)] async fn test_get_logs_from_store_with_filesystem() { use futures::StreamExt; use object_store::{path::Path, ObjectStore, PutPayload}; diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index af9ca957b9..a3f7154cda 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result // A dbt run records the relations it builds, so it looks like a producer // here — but dbt does not trigger downstream runs. Its own DAG is dbt's to // order; the only thing a cascade would add is waking Windmill scripts that - // read a mart, and nothing outside dbt can declare a `dbt://` write, so - // that edge exists in one direction only. Cascading from a project whose - // per-run selection can build any subset of itself needs a per-run write set - // to be correct, which is a design worth doing deliberately rather than - // inferring. Until then dbt materializes and reports; it does not dispatch. + // read a mart. Cascading from a project whose per-run selection can build any + // subset of itself needs a per-run write set to be correct, which is a design + // worth doing deliberately rather than inferring: the deploy-time write set is + // not what ran, and the per-relation state table keeps one row per relation. + // Until then dbt materializes and reports; it does not dispatch. The opposite + // direction does: a native `// materialize manual dbt://…` script reaches the + // fan-out below on the ordinary path, on the strength of its own asset rows. if job.script_lang == Some(ScriptLang::Dbt) { return Ok(DispatchResult::default()); } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 5bb9a1120f..ab91d54287 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -654,6 +654,16 @@ pub struct ResultMetadata { pub wm_failure: Option, } +/// Parses a marker struct out of a job result, which only an object can carry. +/// A derived `Deserialize` also accepts an array, filling fields by position, so +/// without the check a result like `[[], "boom"]` reads as `wm_failure: "boom"`. +pub fn parse_result_object(result: &str) -> Option { + if !result.trim_start().starts_with('{') { + return None; + } + serde_json::from_str(result).ok() +} + /// Sentinel `error.name` we inject into a result when retagging a successful /// run as a failure due to `wm_failure`. Used downstream to detect that /// the result is already in the standard `{ error: { name, message }, ... }` @@ -674,8 +684,7 @@ pub fn is_pre_shaped_wm_failure_result(result: &str) -> bool { struct NameOnly { name: String, } - serde_json::from_str::(result) - .ok() + parse_result_object::(result) .and_then(|m| m.error) .map(|e| e.name == MANUAL_FAILURE_ERROR_NAME) .unwrap_or(false) @@ -721,7 +730,7 @@ impl ValidableJson for Box { } fn result_metadata(&self) -> ResultMetadata { - serde_json::from_str::(self.get()).unwrap_or_default() + parse_result_object::(self.get()).unwrap_or_default() } fn size(&self) -> usize { @@ -774,6 +783,10 @@ impl ValidableJson for serde_json::Value { } fn result_metadata(&self) -> ResultMetadata { + // An array would decode positionally, see `parse_result_object`. + if !self.is_object() { + return ResultMetadata::default(); + } serde_json::from_value::(self.clone()).unwrap_or_default() } @@ -983,7 +996,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64, Option), Error> { +) -> Result<(Uuid, i64), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -1017,7 +1030,7 @@ pub async fn add_completed_job( }; let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| { + let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| { commit_completed_job( db, completed_job, @@ -1052,9 +1065,13 @@ pub async fn add_completed_job( .sleep(tokio::time::sleep) .await?; + if wac_parent_ready { + windmill_common::wac::WAC_SUSPEND_READY.store(true, std::sync::atomic::Ordering::Relaxed); + } + // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration, None)); + return Ok((job_id, duration)); } // Auto-resolve a retry chain that ultimately worked, from whichever of the two @@ -1101,7 +1118,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration, wac_job_ids)) + Ok((completed_job.id, duration)) } async fn commit_completed_job( @@ -1119,7 +1136,7 @@ async fn commit_completed_job( // True when a native script retry was enqueued for this failed attempt, i.e. // this is not the terminal attempt — schedule completion handlers must wait. retry_pending: bool, -) -> windmill_common::error::Result<(Option, i64, bool, Option)> { +) -> windmill_common::error::Result<(Option, i64, bool, bool)> { // let start = std::time::Instant::now(); let job_id = completed_job.id; @@ -1249,74 +1266,23 @@ async fn commit_completed_job( .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?; } - let mut wac_job_ids: Option = None; + // Before `delete_job`: the parent's rows are locked ahead of the child's own + // queue row (see `record_child_completion` for the order this must keep). + let mut wac_parent_ready = false; if !completed_job.is_flow_step() { if let Some(parent_job) = completed_job.parent_job { - // Only update WAC parents (v1 or v2). The WHERE condition skips - // non-WAC parents entirely (error handlers, run_script children, etc.). - // Also returns pending_steps.job_ids so WAC v2 child completion - // doesn't need a separate read. - let row = sqlx::query_scalar!( - r#"UPDATE v2_job_status SET - workflow_as_code_status = jsonb_set( - jsonb_set( - workflow_as_code_status, - array[$1], - COALESCE(workflow_as_code_status->$1, '{}'::jsonb) - ), - array[$1, 'duration_ms'], - to_jsonb($2::bigint) - ) - WHERE id = $3 AND workflow_as_code_status IS NOT NULL - RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#, - &completed_job.id.to_string(), + wac_parent_ready = windmill_common::wac::record_child_completion( + &mut tx, + &parent_job, + &completed_job.id, + success, duration, - parent_job + sanitized_result.as_ref(), ) - .fetch_optional(&mut *tx) .warn_after_seconds(10) - .await - .inspect_err(|e| { - tracing::error!( - "Could not update parent job `duration_ms` in workflow as code status: {}", - e, - ) - }) - .ok() - .flatten(); - wac_job_ids = row.flatten(); - - // If parent was already completed (e.g. cancelled), update v2_job_completed instead - if wac_job_ids.is_none() { - let _ = sqlx::query!( - r#"UPDATE v2_job_completed SET - workflow_as_code_status = jsonb_set( - jsonb_set( - workflow_as_code_status, - array[$1], - COALESCE(workflow_as_code_status->$1, '{}'::jsonb) - ), - array[$1, 'duration_ms'], - to_jsonb($2::bigint) - ) - WHERE id = $3 AND workflow_as_code_status IS NOT NULL"#, - &completed_job.id.to_string(), - duration, - parent_job - ) - .execute(&mut *tx) - .warn_after_seconds(10) - .await - .inspect_err(|e| { - tracing::error!( - "Could not update completed parent job `duration_ms` in workflow as code status: {}", - e, - ) - }); - } + .await?; } } - // tracing::error!("Added completed job {:#?}", queued_job); let mut _skip_downstream_error_handlers = false; tx = delete_job(tx, &job_id).warn_after_seconds(10).await?; @@ -1544,14 +1510,19 @@ async fn commit_completed_job( completed_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids)) + Ok(( + None, + duration, + _skip_downstream_error_handlers, + wac_parent_ready, + )) } async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool, Option), Error>> { +) -> Option, i64, bool, bool), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { @@ -5505,6 +5476,8 @@ async fn push_inner<'c, 'd>( ) { // Check current usage with SELECT (fast, no row locks) // Only check user usage for non-premium workspaces + // `email` here and in the per-user checks below can be a cached dispatch address, up + // to one notify poll stale; accepted, see `get_email_from_permissioned_as`. let (current_workspace_usage, current_user_usage) = check_usage_limits(db, &billing_w_id, email, !team_plan_status.premium).await?; @@ -6894,7 +6867,11 @@ async fn push_inner<'c, 'd>( language as Option, same_worker, pre_run_error.map(|e| e.to_string()), - email, + // `job_authed`'s, not the handed-in `email`: unless the caller's own authed already names + // this identity, it came through `fetch_authed_from_permissioned_as`, which re-resolves the + // address from the principal's live binding. The same statement writes it to + // `job_perms.email`, and the two columns naming different accounts is what this prevents. + job_authed.email, visible_to_owner, flow_innermost_root_job, guarded_concurrent_limit, @@ -7011,7 +6988,8 @@ async fn push_inner<'c, 'd>( hm.insert("created_by", user); } let audit_author = AuditAuthor { - email: email.to_string(), + // `job_authed`'s address, matching `v2_job` and `job_perms` above. + email: job_authed.email.clone(), username: if runs_on_behalf { windmill_common::auth::permissioned_as_to_username(&permissioned_as) } else { @@ -7911,3 +7889,36 @@ mod git_sync_concurrency_key_tests { assert!(a.len() <= 255 && b.len() <= 255); } } + +#[cfg(test)] +mod result_metadata_tests { + use super::{ResultMetadata, ValidableJson}; + use serde_json::value::RawValue; + + fn from_raw(json: &str) -> ResultMetadata { + RawValue::from_string(json.to_string()) + .unwrap() + .result_metadata() + } + + fn from_value(json: &str) -> ResultMetadata { + serde_json::from_str::(json) + .unwrap() + .result_metadata() + } + + #[test] + fn array_result_carries_no_markers() { + for json in [r#"[["label"], "boom"]"#, r#"[null, "boom"]"#] { + for meta in [from_raw(json), from_value(json)] { + assert!( + meta.wm_labels.is_none() && meta.wm_failure.is_none(), + "{json}" + ); + } + } + let meta = from_raw(r#"{"wm_labels": ["label"], "wm_failure": "boom"}"#); + assert_eq!(meta.wm_labels, Some(vec!["label".to_string()])); + assert_eq!(meta.wm_failure.as_deref(), Some("boom")); + } +} diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index 6e80b1405e..d44213d3bc 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -26,7 +26,7 @@ use std::{ cell::RefCell, path::PathBuf, rc::Rc, - sync::{Arc, Mutex}, + sync::{Arc, LazyLock, Mutex}, }; // Re-export deno_telemetry for use by windmill-worker's otel proxy @@ -182,10 +182,33 @@ struct LogString { pub s: mpsc::UnboundedSender, } -#[derive(Clone)] +#[derive(Clone, Default)] pub struct NativeAnnotation { pub useragent: Option, pub proxy: Option<(String, Option<(String, String)>)>, + /// `//fetch_response_timeout `: per-script override of + /// [`default_fetch_response_timeout_secs`]. `Some(0)` disables it for this + /// script; `None` leaves the default in force. + pub fetch_response_timeout_secs: Option, +} + +/// How long `fetch()` waits for a response to begin, in seconds; `0` disables. +/// +/// Covers everything up to the response headers and stops there, so a body may +/// then stream for any length of time. `src/runtime.js` holds the semantics. +/// +/// Must exceed `TIMEOUT_WAIT_RESULT` (default 600), which holds synchronous job +/// calls open without headers. Raising that hot-reloaded instance setting may +/// also require raising `WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS` in the deployment +/// and restarting workers: this environment value is cached for the process. +pub fn default_fetch_response_timeout_secs() -> u64 { + static SECS: LazyLock = LazyLock::new(|| { + std::env::var("WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS") + .ok() + .and_then(|x| x.trim().parse::().ok()) + .unwrap_or(900) + }); + *SECS } /// Serializes V8 isolate creation as defense-in-depth against concurrent @@ -401,7 +424,7 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { } pub fn get_annotation(inner_content: &str) -> NativeAnnotation { - let mut res = NativeAnnotation { useragent: None, proxy: None }; + let mut res = NativeAnnotation::default(); let anns = inner_content .lines() @@ -414,6 +437,13 @@ pub fn get_annotation(inner_content: &str) -> NativeAnnotation { res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string()); } else if ann.starts_with("proxy") { res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim()); + } else if ann.starts_with("fetch_response_timeout") { + // A typo falls back to the default, never to "no timeout". + res.fetch_response_timeout_secs = ann + .trim_start_matches("fetch_response_timeout") + .trim() + .parse::() + .ok(); } } res @@ -554,6 +584,16 @@ pub(crate) fn create_nativets_runtime( let ops = vec![op_get_static_args(), op_log()]; let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; + // deno_web's setTimeout puts its delay through `webidl.converters.long`, + // which wraps at 32 bits: past i32::MAX ms (~24.8 days) the delay comes out + // negative and fires immediately, so an over-generous setting would abort + // every fetch on the spot. Cap rather than wrap. + let fetch_response_timeout_ms = ann + .fetch_response_timeout_secs + .unwrap_or_else(default_fetch_response_timeout_secs) + .saturating_mul(1000) + .min(i32::MAX as u64); + let fetch_options = deno_fetch::Options { root_cert_store_provider: NATIVE_ROOT_CERT_STORE_PROVIDER.clone(), user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), @@ -625,10 +665,15 @@ pub(crate) fn create_nativets_runtime( } // Per-isolate JS init that can't run in the snapshot (runtime.js executes at - // snapshot-build time): currently seeds performance.timeOrigin via - // setTimeOrigin(), which must read this isolate's wall clock. + // snapshot-build time): the wall clock behind performance.timeOrigin and the + // fetch response timeout are both per-isolate values. js_runtime - .execute_script("", "globalThis.__wmInitPerIsolate()") + .execute_script( + "", + format!( + "globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms} }})" + ), + ) .map_err(windmill_common::error::to_anyhow)?; Ok(CreatedRuntime { js_runtime, log_receiver, memory_limit_rx }) diff --git a/backend/windmill-runtime-nativets/src/runtime.js b/backend/windmill-runtime-nativets/src/runtime.js index 01bec09983..88e10f765a 100644 --- a/backend/windmill-runtime-nativets/src/runtime.js +++ b/backend/windmill-runtime-nativets/src/runtime.js @@ -30,9 +30,115 @@ import * as performance from "ext:deno_web/15_performance.js"; import "ext:deno_web/16_image_data.js"; import "ext:deno_fetch/27_eventsource.js"; +// deno_fetch applies no deadline, so a peer that accepts a request and then +// never answers leaves `await fetch(...)` pending until the job timeout, which +// self-hosted defaults to 7 days. +const ORIGINAL_FETCH = fetch.fetch; + +// deno_web's timers reject any `this` other than undefined/globalThis, so +// `timers.setTimeout(...)` passes the module namespace and throws "Illegal +// invocation". +const setTimeoutUnbound = timers.setTimeout; +const clearTimeoutUnbound = timers.clearTimeout; +// Captured before user code shares the isolate and could redefine them, the +// way deno's own modules reach intrinsics through primordials. +const PromiseReject = Promise.reject.bind(Promise); +const promiseThen = Function.prototype.call.bind(Promise.prototype.then); +const abortSignalAny = abortSignal.AbortSignal.any.bind(abortSignal.AbortSignal); +const abortControllerAbort = Function.prototype.call.bind( + abortSignal.AbortController.prototype.abort, +); +const ReflectApply = Reflect.apply; + +// Installed per isolate by __wmInitPerIsolate; 0 disables. Only a backstop +// for the impossible case of fetch running before that init. +let fetchResponseTimeoutMs = 900_000; + +function fetchResponseTimeoutError(requestUrl, timeoutMs) { + let target; + try { + const parsed = new url.URL(requestUrl); + // Query and fragment routinely carry tokens, and this reaches a job log. + target = parsed.origin + parsed.pathname; + } catch { + target = "the request target"; + } + return new domException.DOMException( + `fetch to ${target} timed out: no response headers arrived within ` + + `${Math.round(timeoutMs / 1000)}s (this covers connect, request upload ` + + `and the wait for the server to start replying; once a response begins ` + + `it is never interrupted). Change it per script with ` + + `"//fetch_response_timeout " (0 disables), or instance-wide ` + + `with WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS.`, + "TimeoutError", + ); +} + globalThis.atob = base64.atob; globalThis.btoa = base64.btoa; -globalThis.fetch = fetch.fetch; +// Not `async`, for the same reason deno_fetch's own outer fetch isn't: WPT +// pins that an aborted fetch settles in the same tick, which adopting its +// promise through another one would break. Construction still has to reject +// rather than throw, so it is caught and handed back as a rejection. +globalThis.fetch = function fetch(input, init = undefined) { + const timeoutMs = fetchResponseTimeoutMs; + // Forwarded with the original argument count, so deno still sees an empty + // call as empty and raises its own "1 argument required". The default on + // `init` is what keeps `fetch.length` at 1, as the standard has it. + if (!(timeoutMs > 0) || arguments.length < 1) { + return ReflectApply(ORIGINAL_FETCH, undefined, arguments); + } + + let req; + let controller; + let signal; + try { + // RequestInit is a WebIDL dictionary: copying it drops inherited and + // non-enumerable members, and inheriting from it runs accessors against the + // wrong receiver. Hand it to the same Request constructor fetch() would, + // and carry our own signal in an init of our own. + req = new request.Request(input, init); + + // `req.signal` is deno's own resolution of init.signal over an input + // Request's signal, so combining with it preserves the caller's abort and + // reason while ours only adds a ceiling. + controller = new abortSignal.AbortController(); + signal = abortSignalAny([req.signal, controller.signal]); + } catch (e) { + return PromiseReject(e); + } + + // Already aborted: hand back deno's own settled rejection untouched, and arm + // nothing -- there is no response to wait for. + if (signal.aborted) { + return ORIGINAL_FETCH(req, { signal }); + } + + let timer = setTimeoutUnbound(() => { + timer = undefined; + abortControllerAbort(controller, fetchResponseTimeoutError(req.url, timeoutMs)); + }, timeoutMs); + // Disarmed on headers, never on body completion: a response that has begun + // arriving must be free to stream for as long as it needs. + const disarm = () => { + if (timer !== undefined) { + clearTimeoutUnbound(timer); + timer = undefined; + } + }; + + return promiseThen( + ORIGINAL_FETCH(req, { signal }), + (res) => { + disarm(); + return res; + }, + (e) => { + disarm(); + throw e; + }, + ); +}; globalThis.Request = request.Request; globalThis.Response = response.Response; globalThis.Blob = file.Blob; @@ -123,7 +229,11 @@ Object.assign(globalThis, { // Per-isolate init, invoked from Rust after the snapshot is restored (this // module body runs at snapshot-build time, not per isolate). -globalThis.__wmInitPerIsolate = () => { +globalThis.__wmInitPerIsolate = (config) => { + if (config != null && typeof config.fetchResponseTimeoutMs === "number") { + fetchResponseTimeoutMs = config.fetchResponseTimeoutMs; + } + // setTimeOrigin() seeds performance.timeOrigin from the isolate's wall clock; // without it timeOrigin is undefined and `timeOrigin + performance.now()` is NaN. performance.setTimeOrigin(); diff --git a/backend/windmill-runtime-nativets/src/smoke_tests.rs b/backend/windmill-runtime-nativets/src/smoke_tests.rs index b581a9f1fe..f3da460227 100644 --- a/backend/windmill-runtime-nativets/src/smoke_tests.rs +++ b/backend/windmill-runtime-nativets/src/smoke_tests.rs @@ -24,7 +24,7 @@ use crate::{transpile_ts, NativeAnnotation, PrewarmedIsolate, PrewarmedResult}; /// positional args, and return the isolate's result + captured logs. async fn run_ts(ts: &str, arg_names: &[&str], args: serde_json::Value) -> PrewarmedResult { let js = transpile_ts(ts.to_string()).expect("transpile_ts failed"); - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let arg_names: Vec = arg_names.iter().map(|s| s.to_string()).collect(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, arg_names, None); iso.wait_ready().await.expect("isolate failed to pre-warm"); @@ -232,7 +232,7 @@ export async function main(i: number): Promise { for i in 0..N { let js = js.clone(); let h = tokio::spawn(async move { - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec!["i".to_string()], None); iso.wait_ready().await.expect("pre-warm failed"); diff --git a/backend/windmill-runtime-nativets/tests/annotation.rs b/backend/windmill-runtime-nativets/tests/annotation.rs new file mode 100644 index 0000000000..d8319a87a5 --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/annotation.rs @@ -0,0 +1,37 @@ +//! `//fetch_response_timeout` parsing. Cheap (no V8), so it stays out of the +//! e2e file. + +use windmill_runtime_nativets::get_annotation; + +#[test] +fn a_value_is_parsed_and_zero_stays_distinct_from_absent() { + // Collapsing `Some(0)` to `None` would silently reinstate the default on a + // script that explicitly asked for no limit. + assert_eq!( + get_annotation("//native\n//fetch_response_timeout 30\n").fetch_response_timeout_secs, + Some(30) + ); + assert_eq!( + get_annotation("//fetch_response_timeout 0\n").fetch_response_timeout_secs, + Some(0) + ); + assert_eq!( + get_annotation("//native\n").fetch_response_timeout_secs, + None + ); +} + +#[test] +fn a_malformed_value_falls_back_to_the_default_not_to_no_timeout() { + for src in [ + "//fetch_response_timeout abc\n", + "//fetch_response_timeout\n", + "//fetch_response_timeout -5\n", + ] { + assert_eq!( + get_annotation(src).fetch_response_timeout_secs, + None, + "{src:?} should leave the default in force" + ); + } +} diff --git a/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs b/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs new file mode 100644 index 0000000000..5118b5f93f --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs @@ -0,0 +1,593 @@ +//! The nativets `fetch()` response timeout. +//! +//! Two halves of one contract, and a fix that satisfies only the first is worse +//! than no fix: +//! +//! 1. a peer that accepts a request and never answers is given up on +//! 2. a response that has begun arriving is never cut off, however long it +//! takes in total +//! +//! (2) rules out the obvious implementation — `AbortSignal.timeout(N)` around +//! every fetch would satisfy (1) and break every streaming response and long +//! download. +//! +//! Hermetic: loopback listeners, no egress. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::Mutex; + +use windmill_runtime_nativets::{transpile_ts, NativeAnnotation, PrewarmedIsolate}; + +/// The annotation is in whole seconds, so the tests scale around this. +const TIMEOUT_SECS: u64 = 2; + +async fn run_with_timeout_secs(ts: &str, secs: u64) -> Result { + let js = transpile_ts(ts.to_string()).expect("transpile_ts failed"); + let ann = + NativeAnnotation { fetch_response_timeout_secs: Some(secs), ..NativeAnnotation::default() }; + let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None); + iso.wait_ready().await.expect("isolate failed to pre-warm"); + let res = iso + .start_execution("{}".to_string()) + .wait() + .await + .expect("isolate panicked"); + res.result.map(|raw| raw.get().to_string()) +} + +/// A peer that reads the request and then answers nothing, closing only after +/// `close_after` so no test leaves an isolate wedged on a pending fetch. +/// +/// The socket is held rather than dropped on accept: dropping it sends a FIN, +/// which surfaces as a connection error — the easy failure, not this one. +async fn spawn_silent_peer(seen: Arc>>, close_after: Duration) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let seen = seen.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + if let Ok(n) = sock.read(&mut buf).await { + seen.lock().await.extend_from_slice(&buf[..n]); + } + tokio::time::sleep(close_after).await; + drop(sock); + }); + } + }); + port +} + +/// A peer that records the request it received and answers 200 immediately. +async fn spawn_echo_peer(seen: Arc>>) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let seen = seen.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + if let Ok(n) = sock.read(&mut buf).await { + seen.lock().await.extend_from_slice(&buf[..n]); + } + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + }); + } + }); + port +} + +/// A peer that responds after `headers_after`, then dribbles a chunked body out +/// over `chunks * chunk_every`. +async fn spawn_streaming_peer( + headers_after: Duration, + chunks: usize, + chunk_every: Duration, +) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + let _ = sock.read(&mut buf).await; + tokio::time::sleep(headers_after).await; + if sock + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n", + ) + .await + .is_err() + { + return; + } + for _ in 0..chunks { + tokio::time::sleep(chunk_every).await; + if sock.write_all(b"1\r\nx\r\n").await.is_err() { + return; + } + } + let _ = sock.write_all(b"0\r\n\r\n").await; + }); + } + }); + port +} + +const POST_TO_SILENT_PEER: &str = r#" +export async function main(): Promise { + const res = await fetch("http://127.0.0.1:{port}/orders", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: 1 }), + }); + return res.status; +} +"#; + +fn post_script(port: u16) -> String { + POST_TO_SILENT_PEER.replace("{port}", &port.to_string()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_peer_that_never_answers_is_given_up_on() { + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen.clone(), Duration::from_secs(60)).await; + + let started = Instant::now(); + let err = run_with_timeout_secs(&post_script(port), TIMEOUT_SECS) + .await + .expect_err("fetch against a peer that never answers must not resolve"); + let elapsed = started.elapsed(); + + assert!( + elapsed >= Duration::from_secs(TIMEOUT_SECS), + "gave up after {elapsed:?}, before the configured {TIMEOUT_SECS}s -- \ + the timeout is firing on something other than the wait for a response", + ); + assert!( + elapsed < Duration::from_secs(TIMEOUT_SECS + 15), + "took {elapsed:?} to give up", + ); + + // The message has to stand on its own in a job log: a hung request is + // otherwise indistinguishable from a slow one. + assert!( + err.contains("no response headers arrived within"), + "error should explain what timed out, got: {err}", + ); + assert!( + err.contains("/orders") && err.contains("fetch_response_timeout"), + "error should name the target and how to change the limit, got: {err}", + ); + + // Without this the test would also pass if the request never went out. + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.contains("POST /orders"), + "peer should have received the request, got: {body}", + ); +} + +/// The counterfactual for the test above: with the timeout disabled, the same +/// script against the same peer is still running well past the point the +/// timeout would have fired. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn without_a_timeout_the_same_request_keeps_running() { + let seen = Arc::new(Mutex::new(Vec::new())); + // Closes eventually, so the isolate unwinds instead of pinning a blocking + // task for the rest of the test binary's life. + let port = spawn_silent_peer(seen, Duration::from_secs(TIMEOUT_SECS * 3)).await; + + let still_running = tokio::time::timeout( + Duration::from_secs(TIMEOUT_SECS * 2), + run_with_timeout_secs(&post_script(port), 0), + ) + .await + .is_err(); + + assert!( + still_running, + "with the timeout disabled the request should still have been pending \ + at {}s -- if it ends on its own, the test above proves nothing", + TIMEOUT_SECS * 2, + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_streaming_body_outliving_the_timeout_is_not_cut_off() { + // Headers land fast, then the body trickles well past the timeout. A + // total-duration timeout fails here; that is the point of the test. + let port = + spawn_streaming_peer(Duration::from_millis(200), 10, Duration::from_millis(500)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/stream"); + return `${{res.status}}:${{(await res.text()).length}}`; +}} +"# + ); + + let started = Instant::now(); + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a streaming response must not be interrupted"); + let elapsed = started.elapsed(); + + assert_eq!(out, "\"200:10\"", "full body should arrive intact"); + assert!( + elapsed > Duration::from_secs(TIMEOUT_SECS), + "the transfer ({elapsed:?}) has to outlast the {TIMEOUT_SECS}s timeout \ + for this to be exercising anything", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_slow_but_answering_peer_is_not_cut_off() { + // A quarter of the window rather than half: CI runs this at + // --test-threads=10 alongside other V8 isolates, and this margin is what + // absorbs executor starvation. + let port = spawn_streaming_peer( + Duration::from_millis(1_000 * TIMEOUT_SECS / 4), + 1, + Duration::from_millis(10), + ) + .await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/slow"); + await res.text(); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a slow but answering peer must not be cut off"); + assert_eq!(out, "200"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_over_large_timeout_does_not_wrap_into_an_instant_one() { + // deno_web's setTimeout puts its delay through `webidl.converters.long`, + // which wraps at 32 bits. Unclamped, this ~46-day setting wraps negative and + // aborts immediately -- asking for a longer leash would kill every fetch. + let port = spawn_streaming_peer(Duration::from_millis(50), 1, Duration::from_millis(10)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/ok"); + await res.text(); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, 4_000_000) + .await + .expect("an over-large timeout must not abort the request"); + assert_eq!(out, "200"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_caller_abort_still_wins_with_its_own_reason() { + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(60)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const ac = new AbortController(); + setTimeout(() => ac.abort(new Error("caller_abort_marker")), 200); + try {{ + await fetch("http://127.0.0.1:{port}/probe", {{ signal: ac.signal }}); + return "unexpectedly resolved"; + }} catch (e) {{ + return String((e as Error).message); + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch its own abort"); + assert!( + out.contains("caller_abort_marker"), + "caller's abort reason should survive being combined with ours, got: {out}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_timeout_is_catchable_as_a_timeout_error() { + // Scripts that retry on transient failures need to recognise this one; + // `TimeoutError` matches AbortSignal.timeout()'s reason. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(60)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + try {{ + await fetch("http://127.0.0.1:{port}/probe"); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch the timeout"); + assert_eq!(out, "\"TimeoutError\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_init_whose_members_are_inherited_is_not_flattened() { + // RequestInit is a WebIDL dictionary and deno_fetch reads its members with + // plain property gets, so they may sit on the prototype chain or be + // non-enumerable. Object spread copies neither, which would silently + // downgrade this POST to a GET and drop the header. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +declare const Object: any; +export async function main(): Promise {{ + const base = {{ method: "POST", headers: {{ "x-probe": "yes" }} }}; + const res = await fetch("http://127.0.0.1:{port}/inherited", Object.create(base)); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("request should succeed"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("POST /inherited"), + "inherited `method` should survive, got: {body}", + ); + assert!( + body.to_lowercase().contains("x-probe: yes"), + "inherited `headers` should survive, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_dictionary_init_still_fails_loudly() { + // deno's dictionary converter throws on a non-object init. Copying members + // into a fresh object instead of inheriting would turn `"POST"` into + // {0:"P",1:"O",...} -- a valid dictionary with ignored keys, i.e. a silent + // GET where the caller used to get a TypeError. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + try {{ + await fetch("http://127.0.0.1:{port}/x", "POST" as any); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch the error"); + assert_eq!(out, "\"TypeError\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_accessor_backed_init_reads_against_its_own_receiver() { + // A getter on the init must run with the object it was defined on as `this`, + // or a private field is unreachable and it throws. Carrying the init across + // by inheritance rather than by handing it to Request would break this. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +class Init {{ + #method = "POST"; + #body = "from-private-field"; + get method(): string {{ return this.#method; }} + get body(): string {{ return this.#body; }} +}} +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/accessor", new Init() as any); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("an accessor-backed init must not throw"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("POST /accessor") && body.contains("from-private-field"), + "getter-provided method and body should both reach the wire, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_request_input_keeps_its_body_and_headers() { + // The wrapper builds a Request and hands that to fetch, so the body survives + // one more construction than it used to -- deno proxies it rather than + // consuming it, and this pins that. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const req = new Request("http://127.0.0.1:{port}/from-request", {{ + method: "PUT", + headers: {{ "x-probe": "yes" }}, + body: "payload-body", + }}); + const res = await fetch(req); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a Request input must work"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("PUT /from-request") + && body.to_lowercase().contains("x-probe: yes") + && body.contains("payload-body"), + "method, headers and body should all survive, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_already_aborted_fetch_settles_in_the_same_tick() { + // deno_fetch keeps its outer fetch non-async and returns an already-settled + // rejection untouched, because WPT pins that an aborted fetch settles in the + // same tick. Adopting it through another promise pushes the rejection behind + // any microtask queued after the call. + let ts = r#" +export async function main(): Promise { + const order: string[] = []; + const ac = new AbortController(); + ac.abort(); + const f = fetch("http://127.0.0.1:1/x", { signal: ac.signal }) + .catch(() => { order.push("fetch"); }); + Promise.resolve().then(() => { order.push("queued-after"); }); + await f; + await new Promise((r) => setTimeout(r, 0)); + return order.join(","); +} +"#; + + let out = run_with_timeout_secs(ts, TIMEOUT_SECS) + .await + .expect("script should run"); + assert_eq!( + out, "\"fetch,queued-after\"", + "the rejection must land before a microtask queued after the call", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_wrapper_keeps_fetch_s_own_shape() { + // The wrapper is indistinguishable from deno's fetch on three counts a + // script can observe: its arity, the error for an empty call, and not + // depending on a mutable `Promise.prototype.then` the way an ordinary + // property lookup would. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen).await; + + let ts = format!( + r#" +declare const Promise: any; +declare const AbortSignal: any; +declare const AbortController: any; +export async function main(): Promise {{ + const arity = (fetch as any).length; + + // The message, not just the type: forwarding two explicit `undefined`s + // would still throw a TypeError, just deno's invalid-URL one instead of + // its required-argument one. + let emptyCall = "resolved"; + try {{ + await (fetch as any)(); + }} catch (e) {{ + emptyCall = (e as Error).message.includes("1 argument required") + ? "required-argument" + : `other(${{(e as Error).message}})`; + }} + + // Patched only across the call: the wrapper reaches for these while + // building its return value, and awaiting under a patched Promise + // prototype would instead measure V8 treating it as a plain thenable. + const originalThen = Promise.prototype.then; + const originalAny = AbortSignal.any; + const originalAbort = AbortController.prototype.abort; + let pending: any; + try {{ + Promise.prototype.then = undefined; + (AbortSignal as any).any = undefined; + (AbortController.prototype as any).abort = undefined; + pending = fetch("http://127.0.0.1:{port}/shape"); + }} finally {{ + Promise.prototype.then = originalThen; + (AbortSignal as any).any = originalAny; + (AbortController.prototype as any).abort = originalAbort; + }} + const status = (await pending).status; + + return `${{arity}}:${{emptyCall}}:${{status}}`; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should run"); + assert_eq!(out, "\"1:required-argument:200\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_timer_aborts_through_a_captured_intrinsic() { + // The timeout has to fire for this one: `AbortController.prototype.abort` + // is patched out for the whole wait, so an ordinary lookup would throw + // inside the timer callback and leave the request pending forever. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(TIMEOUT_SECS * 5)).await; + + let ts = format!( + r#" +declare const AbortController: any; +export async function main(): Promise {{ + const originalAbort = AbortController.prototype.abort; + AbortController.prototype.abort = undefined; + try {{ + await fetch("http://127.0.0.1:{port}/patched-abort"); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} finally {{ + AbortController.prototype.abort = originalAbort; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("the timeout must still fire"); + assert_eq!(out, "\"TimeoutError\""); +} diff --git a/backend/windmill-runtime-nativets/tests/otel_e2e.rs b/backend/windmill-runtime-nativets/tests/otel_e2e.rs index cadddb2fcf..22fd18de90 100644 --- a/backend/windmill-runtime-nativets/tests/otel_e2e.rs +++ b/backend/windmill-runtime-nativets/tests/otel_e2e.rs @@ -136,7 +136,7 @@ export async function main(): Promise {{ "# ); let js = transpile_ts(ts).expect("transpile failed"); - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None); iso.wait_ready().await.expect("isolate failed to pre-warm"); diff --git a/backend/windmill-runtime-nativets/tests/response_timeout_env.rs b/backend/windmill-runtime-nativets/tests/response_timeout_env.rs new file mode 100644 index 0000000000..a206c63134 --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/response_timeout_env.rs @@ -0,0 +1,13 @@ +//! Its own test binary: the setting is read once per process through a +//! `LazyLock`, so nothing else may resolve it first. Adding a second test to +//! this file breaks that isolation. + +use windmill_runtime_nativets::default_fetch_response_timeout_secs; + +#[test] +fn the_env_var_is_what_operators_actually_set() { + // A typo in the variable's name would compile, pass every other test, and + // silently hand every operator the built-in default instead. + std::env::set_var("WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS", "17"); + assert_eq!(default_fetch_response_timeout_secs(), 17); +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index f9bfe2f4df..5bb15b456e 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -7,7 +7,7 @@ */ use dashmap::DashMap; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::IpAddr; use std::sync::LazyLock; @@ -17,6 +17,7 @@ use windmill_api_auth::{ }; use windmill_common::db::DB; use windmill_common::per_minute_counter::PerMinuteCounter; +use windmill_common::ssrf::{private_git_host_allowed, private_git_host_hint, GitRemoteCaller}; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; @@ -50,9 +51,9 @@ use windmill_common::{ error::{self, Error, JsonResult, Result}, get_database_url, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only, - fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, WithDraftOverlay, - WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only, fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, + WithDraftOverlay, WithDraftQuery, }, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, @@ -89,6 +90,9 @@ pub fn workspaced_service() -> Router { .route("/git_commit_hash/{*path}", get(get_git_commit_hash)) .route("/type/list", get(list_resource_types)) .route("/type/listnames", get(list_resource_types_names)) + .route("/type/resource_counts", get(list_resource_counts_by_type)) + .route("/type/hub/info", get(list_hub_resource_type_info)) + .route("/type/hub/pick/{name}", post(pick_hub_resource_type)) .route("/type/get/{name}", get(get_resource_type)) .route("/type/exists/{name}", get(exists_resource_type)) .route("/type/update/{name}", post(update_resource_type)) @@ -114,6 +118,10 @@ pub struct ResourceType { pub edited_at: Option>, pub format_extension: Option, pub is_fileset: bool, + /// The name the product goes by (`gsheets` is "Google Sheets"), null where nobody named it. + /// Skipped when absent, so the type files of a synced repo gain nothing until one is set. + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, } #[derive(Deserialize)] @@ -123,6 +131,7 @@ pub struct CreateResourceType { pub description: Option, pub format_extension: Option, pub is_fileset: Option, + pub display_name: Option, } #[derive(Deserialize)] @@ -134,8 +143,18 @@ pub struct EditResourceType { /// `Option` conflates: an absent field leaves the extension alone, while an /// explicit `null` clears it. A hub pull relies on both — a type that stops /// being a file type has to stop being one locally too. - #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] pub format_extension: Option>, + /// Doubly optional for the same reason. A push from a CLI that predates the field omits it, + /// and must not clear a name the hub set. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } #[derive(FromRow, Serialize, Deserialize)] @@ -1290,9 +1309,26 @@ async fn create_resource( webhook.send_message( w_id.clone(), - WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() }, + WebhookMessage::CreateResource { workspace: w_id.clone(), path: resource.path.clone() }, ); + // Trigger CI tests for items that reference this resource + { + let db2 = db.clone(); + let path2 = resource.path.clone(); + let email2 = authed.email.clone(); + let username2 = authed.username.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_dep_map::ci_tests::trigger_ci_tests_for_item( + &db2, &w_id, &path2, "resource", &email2, &username2, + ) + .await + { + tracing::error!(%e, "error triggering CI tests after resource creation"); + } + }); + } + Ok(( StatusCode::CREATED, format!("resource {} created", resource.path), @@ -1309,6 +1345,17 @@ async fn delete_resource( let path = path.to_path(); check_scopes(&authed, || format!("resources:write:{}", path))?; + + // Ahead of the deploy rules: nothing is deployed at a draft-only path, so + // gating this discard on them would strand the row in a protected workspace. + // Ahead of the transaction too — the not-found branch other kinds hang this + // off is the `not_found_if_none` below, past the linked-variable cascade. + if delete_draft_only_for_path(&db, &w_id, UserDraftItemKind::Resource, path, &authed.email) + .await? + { + return Ok(format!("draft-only resource {} deleted", path)); + } + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -1320,53 +1367,22 @@ async fn delete_resource( { return Err(Error::PermissionDenied(msg)); } + + let cascade = plan_linked_var_cascade(&db, &w_id, &[path.to_string()]).await?; + let mut tx = user_db.begin(&authed).await?; - // Capture resource data for trashbin before deleting - let trash_resource: Option = sqlx::query_scalar( - "SELECT to_jsonb(t) FROM resource t WHERE path = $1 AND workspace_id = $2", + // The whole row comes back out of the delete, so the trashbin entry below is built from + // what RLS actually removed, and the cascade runs only once RLS has allowed the delete. + let deleted: Option<(String, serde_json::Value)> = sqlx::query_as( + "DELETE FROM resource AS t WHERE t.path = $1 AND t.workspace_id = $2 + RETURNING t.path, to_jsonb(t)", ) .bind(path) .bind(&w_id) .fetch_optional(&mut *tx) .await?; - - // Fetch the resource value before deleting, so we can find linked $var: references - let resource_value: Option> = - sqlx::query_scalar("SELECT value FROM resource WHERE path = $1 AND workspace_id = $2") - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - // Collect all $var: paths referenced in the resource value - let mut linked_var_paths: Vec = Vec::new(); - if let Some(Some(ref value)) = resource_value { - collect_var_refs(value, &mut linked_var_paths); - } - - // A scoped token must not delete linked variables it lacks variables:write for. - check_linked_var_delete_scopes(&authed, &linked_var_paths)?; - - // Capture linked variables for trashbin before deleting them - let trash_linked_vars: Vec = if linked_var_paths.is_empty() { - Vec::new() - } else { - let placeholders: Vec = linked_var_paths - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "SELECT to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path IN ({})", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, serde_json::Value>(&query).bind(&w_id); - for var_path in &linked_var_paths { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; + let (deleted_path, res_data) = not_found_if_none(deleted, "Resource", &path)?; sqlx::query!( "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2", @@ -1376,64 +1392,63 @@ async fn delete_resource( .execute(&mut *tx) .await?; - let deleted_path = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", - path, - w_id + let linked_var_paths = cascade.resolve(std::slice::from_ref(&deleted_path)); + + // A scoped token must not delete linked variables it lacks variables:write for. Erroring + // here rolls the resource delete back with it, so nothing is deleted either way. + check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + + // Capture linked variables for trashbin before deleting them + let trash_linked_vars: Vec<(String, serde_json::Value)> = sqlx::query_as( + "SELECT path, to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path = ANY($2)", ) - .fetch_optional(&mut *tx) + .bind(&w_id) + .bind(&linked_var_paths) + .fetch_all(&mut *tx) .await?; - not_found_if_none(deleted_path, "Resource", &path)?; - // Delete linked variables that are actually referenced in the resource value - let deleted_linked_variables: Vec = if linked_var_paths.is_empty() { - Vec::new() - } else { - // Clean up any ws_specific rows for these variables first - // (mark_linked_variables_ws_specific may have auto-inserted them) so - // they don't survive the variable deletion as orphans — a variable - // later recreated at the same path would otherwise inherit the stale - // ws_specific flag. - sqlx::query!( - "DELETE FROM ws_specific - WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", - w_id, - &linked_var_paths - ) - .execute(&mut *tx) - .await?; + let deleted_linked_variables = sqlx::query_scalar!( + "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + w_id, + &linked_var_paths + ) + .fetch_all(&mut *tx) + .await?; - let placeholders: Vec = linked_var_paths - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "DELETE FROM variable WHERE workspace_id = $1 AND path IN ({}) RETURNING path", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, String>(&query).bind(&w_id); - for var_path in &linked_var_paths { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; + // ws_specific has no FK to variable, so a row mark_linked_variables_ws_specific inserted + // would survive as an orphan and a variable later recreated at that path would inherit + // the stale flag. + sqlx::query!( + "DELETE FROM ws_specific + WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + w_id, + &deleted_linked_variables + ) + .execute(&mut *tx) + .await?; - if let Some(res_data) = trash_resource { - let mut trash_data = serde_json::json!({"row": res_data}); - if !trash_linked_vars.is_empty() { - trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); - } - windmill_common::trashbin::move_to_trash( - &mut *tx, - &w_id, - "resource", - path, - trash_data, - &authed.username, - ) - .await?; + // Only the rows that actually went: the snapshot above is what the caller could read, + // which is not necessarily what RLS let it delete, and the trashbin must not hold a copy + // of a secret that is still live. + let trash_linked_vars: Vec = trash_linked_vars + .into_iter() + .filter(|(var_path, _)| deleted_linked_variables.contains(var_path)) + .map(|(_, row)| row) + .collect(); + + let mut trash_data = serde_json::json!({"row": res_data}); + if !trash_linked_vars.is_empty() { + trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); } + windmill_common::trashbin::move_to_trash( + &mut *tx, + &w_id, + "resource", + path, + trash_data, + &authed.username, + ) + .await?; audit_log( &mut *tx, @@ -1445,6 +1460,24 @@ async fn delete_resource( None, ) .await?; + + // The cascade is the one way a variable dies without a variables/delete request of its + // own, so give each one the audit row it would have had, stamped with what took it. + for var_path in &deleted_linked_variables { + let mut params = HashMap::new(); + params.insert("via_resource", path); + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(var_path), + Some(params), + ) + .await?; + } + tx.commit().await?; // Resource gone for everyone: wipe ALL users' drafts at this path (and any linked @@ -1496,35 +1529,205 @@ async fn delete_resource( ); } - Ok(format!("resource {} deleted", path)) + // Name what else went: the cascade is silent from the caller's side otherwise, and a + // secret it took is not something to discover later from a failing job. + if deleted_linked_variables.is_empty() { + Ok(format!("resource {} deleted", path)) + } else { + Ok(format!( + "resource {} deleted, along with its linked variables: {}", + path, + deleted_linked_variables.join(", ") + )) + } } -/// Recursively collect all `$var:path` references from a JSON value. -fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { +/// The forms that resolve a variable path against `variable`, so a value carrying any of them +/// breaks when that variable goes. Only `$var:` is minted by the resource editor, which is why +/// `collect_var_refs` stays narrower than this. +const REFERRER_PREFIXES: [&str; 2] = ["$var:", "$jsonvar:"]; + +/// Recursively collect the variable paths a JSON value references through any of `prefixes`. +fn collect_refs_with_prefixes(value: &serde_json::Value, prefixes: &[&str], out: &mut Vec) { match value { serde_json::Value::String(s) => { - if let Some(var_path) = s.strip_prefix("$var:") { + if let Some(var_path) = prefixes.iter().find_map(|p| s.strip_prefix(p)) { out.push(var_path.to_string()); } } serde_json::Value::Object(m) => { for v in m.values() { - collect_var_refs(v, out); + collect_refs_with_prefixes(v, prefixes, out); } } serde_json::Value::Array(arr) => { for v in arr { - collect_var_refs(v, out); + collect_refs_with_prefixes(v, prefixes, out); } } _ => {} } } -/// Deleting a resource cascades into the `$var:` variables its value references. A -/// scoped token must not use that cascade to delete variables it could not delete -/// directly via `delete_variable` (which gates on `variables:write:`), so require -/// `variables:write` for EVERY linked variable and fail the whole delete otherwise. +/// Recursively collect all `$var:path` references from a JSON value. +fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { + collect_refs_with_prefixes(value, &["$var:"], out) +} + +/// Whether the variable at `var_path` is the resource's own secret rather than one its value +/// merely points at: the same-path twin `delete_variable` and the `update_resource` rename +/// already act on, or `_`, which the connect form mints for a resource +/// type with several secret fields. Anything else is a standalone workspace variable, and +/// deleting one destroys a secret its other referrers still need. +/// +/// A rename moves only the twin, so `_` secrets stop matching and are left +/// behind instead. An orphaned secret can be deleted by hand; a destroyed one cannot. +fn is_owned_linked_var(resource_path: &str, var_path: &str) -> bool { + var_path == resource_path + || var_path + .strip_prefix(resource_path) + .is_some_and(|suffix| suffix.starts_with('_')) +} + +/// Which of `var_paths` a resource outside `excluded_resource_paths` still references. +/// +/// Must run off the non-RLS pool: a referrer in a folder the caller cannot read is precisely +/// the one whose variable has to survive. Nothing from those rows reaches the response. +async fn linked_vars_referenced_elsewhere( + db: &DB, + w_id: &str, + var_paths: &[String], + excluded_resource_paths: &[String], +) -> Result> { + if var_paths.is_empty() { + return Ok(HashSet::new()); + } + // The quotes around the pattern are what make it a whole-JSON-string match rather than a + // prefix one, so `f/db` does not match `"$var:f/db_replica"`. Paths are `proper_id` + // segments, so no JSON escaping or LIKE wildcard can reach this. + let referenced = sqlx::query_scalar!( + "WITH survivors AS ( + SELECT value::text AS rendered FROM resource + WHERE workspace_id = $1 AND NOT (path = ANY($2::text[])) + ) + SELECT v.path FROM unnest($3::text[]) AS v(path) + WHERE EXISTS ( + SELECT 1 FROM survivors s + WHERE strpos(s.rendered, '\"$var:' || v.path || '\"') > 0 + OR strpos(s.rendered, '\"$jsonvar:' || v.path || '\"') > 0 + )", + w_id, + excluded_resource_paths, + var_paths, + ) + .fetch_all(db) + .await?; + Ok(referenced.into_iter().flatten().collect()) +} + +/// A resource delete's candidate cascade, gathered before the caller's transaction opens: the +/// referrer scan runs on `db` because it has to see resources RLS hides, and a second acquire +/// from that same pool under an open `user_db` transaction stalls to the acquire timeout when +/// `DATABASE_CONNECTIONS` is small. `resolve` then decides without touching the database. +/// +/// So a resource that starts referencing a candidate between the scan and the delete keeps a +/// `$var:` pointing at nothing. Narrowing that window means running the scan on the +/// transaction's own connection under a tightly scoped `SET LOCAL ROLE NONE` (the elevation +/// `windmill-queue/src/schedule.rs` uses); closing it needs a lock on every resource write. +struct LinkedVarCascade { + /// Each owned `$var:` path with the requested resource whose value carries it. + candidates: Vec<(String, String)>, + /// Requested resource paths, each with every variable path its value references. + requested: Vec<(String, Vec)>, + /// Candidate paths a resource outside the requested set still references. + referenced_outside: HashSet, +} + +impl LinkedVarCascade { + /// The variables to delete, now that RLS has settled which resources went. + /// + /// A requested resource left standing is the one referrer the scan could not account for, + /// having had to exclude every requested path before RLS had ruled. + fn resolve(&self, deleted_paths: &[String]) -> Vec { + let referenced_by_survivors: HashSet<&str> = self + .requested + .iter() + .filter(|(path, _)| !deleted_paths.contains(path)) + .flat_map(|(_, refs)| refs.iter().map(String::as_str)) + .collect(); + + let mut resolved: Vec = self + .candidates + .iter() + .filter(|(var_path, owner)| { + deleted_paths.contains(owner) + && !self.referenced_outside.contains(var_path.as_str()) + && !referenced_by_survivors.contains(var_path.as_str()) + }) + .map(|(var_path, _)| var_path.clone()) + .collect(); + resolved.sort(); + resolved.dedup(); + resolved + } + + /// Which deleted resource the cascade took `var_path` for, to stamp on its audit row. + fn owner_of<'a>(&'a self, var_path: &str, deleted_paths: &[String]) -> Option<&'a str> { + self.candidates + .iter() + .find(|(candidate, owner)| candidate == var_path && deleted_paths.contains(owner)) + .map(|(_, owner)| owner.as_str()) + } +} + +/// Gather what `LinkedVarCascade::resolve` needs for a delete of `paths`. +async fn plan_linked_var_cascade( + db: &DB, + w_id: &str, + paths: &[String], +) -> Result { + let rows: Vec<(String, Option)> = sqlx::query_as( + "SELECT path, value FROM resource WHERE workspace_id = $1 AND path = ANY($2)", + ) + .bind(w_id) + .bind(paths) + .fetch_all(db) + .await?; + + let mut candidates: Vec<(String, String)> = Vec::new(); + let mut requested: Vec<(String, Vec)> = Vec::new(); + for (path, value) in rows { + let mut owned: Vec = Vec::new(); + let mut refs: Vec = Vec::new(); + if let Some(value) = &value { + collect_var_refs(value, &mut owned); + collect_refs_with_prefixes(value, &REFERRER_PREFIXES, &mut refs); + } + candidates.extend( + owned + .into_iter() + .filter(|var_path| is_owned_linked_var(&path, var_path)) + .map(|var_path| (var_path, path.clone())), + ); + requested.push((path, refs)); + } + candidates.sort(); + candidates.dedup(); + + let mut candidate_paths: Vec = candidates + .iter() + .map(|(var_path, _)| var_path.clone()) + .collect(); + candidate_paths.dedup(); + let referenced_outside = + linked_vars_referenced_elsewhere(db, w_id, &candidate_paths, paths).await?; + Ok(LinkedVarCascade { candidates, requested, referenced_outside }) +} + +/// Deleting a resource cascades into the `$var:` variables it owns. A scoped token must not +/// use that cascade to delete variables it could not delete directly via `delete_variable` +/// (which gates on `variables:write:`), so require `variables:write` for EVERY cascaded +/// variable and fail the whole delete otherwise. /// /// No co-located-path exemption: a resource and a variable may share a path, and a /// resource-write token can create a resource over an existing standalone variable and @@ -1627,111 +1830,91 @@ async fn delete_resources_bulk( return Err(Error::PermissionDenied(msg)); } + let cascade = plan_linked_var_cascade(&db, &w_id, &request.paths).await?; + let mut tx = user_db.begin(&authed).await?; - // Capture resources for trashbin per path before bulk delete, and - // collect $var: references so we can cascade-delete the linked variables - // (matching single-resource delete semantics). - let mut linked_var_paths: Vec = Vec::new(); - for path in &request.paths { - let trash_resource: Option = sqlx::query_scalar( - "SELECT to_jsonb(t) FROM resource t WHERE path = $1 AND workspace_id = $2", - ) - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - if let Some(res_data) = trash_resource { - // Per-resource linked vars so each resource's trash entry carries - // exactly the variables that vanished with it (matching the - // single-delete shape: trash_data["linked_variables"]). - let mut this_linked: Vec = Vec::new(); - if let Some(value) = res_data.get("value") { - collect_var_refs(value, &mut this_linked); - } - this_linked.sort(); - this_linked.dedup(); - - let trash_linked_vars: Vec = if this_linked.is_empty() { - Vec::new() - } else { - let placeholders: Vec = this_linked - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "SELECT to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path IN ({})", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, serde_json::Value>(&query).bind(&w_id); - for var_path in &this_linked { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; - - let mut trash_data = serde_json::json!({"row": res_data}); - if !trash_linked_vars.is_empty() { - trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); - } - windmill_common::trashbin::move_to_trash( - &mut *tx, - &w_id, - "resource", - path, - trash_data, - &authed.username, - ) - .await?; - - linked_var_paths.extend(this_linked); - } - } - linked_var_paths.sort(); - linked_var_paths.dedup(); - - // A scoped token must not delete linked variables it lacks variables:write for. - check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + // Whole rows out of the delete; see delete_resource. RLS can leave a requested resource + // standing, so everything below is driven by this list rather than by `request.paths`. + let deleted: Vec<(String, serde_json::Value)> = sqlx::query_as( + "DELETE FROM resource AS t WHERE t.path = ANY($1) AND t.workspace_id = $2 + RETURNING t.path, to_jsonb(t)", + ) + .bind(&request.paths) + .bind(&w_id) + .fetch_all(&mut *tx) + .await?; + let deleted_paths: Vec = deleted.iter().map(|(path, _)| path.clone()).collect(); sqlx::query!( "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = ANY($2)", w_id, - &request.paths + &deleted_paths ) .execute(&mut *tx) .await?; - let deleted_paths = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", - &request.paths, - w_id + let linked_var_paths = cascade.resolve(&deleted_paths); + + // A scoped token must not delete linked variables it lacks variables:write for. Erroring + // here rolls the resource deletes back with it. + check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + + // Snapshot before the delete below: the trashbin entries need the rows. + let trash_linked_vars: Vec<(String, serde_json::Value)> = sqlx::query_as( + "SELECT path, to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path = ANY($2)", + ) + .bind(&w_id) + .bind(&linked_var_paths) + .fetch_all(&mut *tx) + .await?; + + let deleted_linked_variables = sqlx::query_scalar!( + "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + w_id, + &linked_var_paths ) .fetch_all(&mut *tx) .await?; - // Cascade-clean linked variables: delete any ws_specific 'variable' rows - // (typically auto-inserted by mark_linked_variables_ws_specific when the - // resource was ws_specific) BEFORE deleting the variable rows themselves - // — otherwise those ws_specific rows survive as orphans and a later - // variable created at the same path would inherit a stale flag. - if !linked_var_paths.is_empty() { - sqlx::query!( - "DELETE FROM ws_specific - WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", - w_id, - &linked_var_paths - ) - .execute(&mut *tx) - .await?; + // See delete_resource: ws_specific has no FK, so the rows would orphan. + sqlx::query!( + "DELETE FROM ws_specific + WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + w_id, + &deleted_linked_variables + ) + .execute(&mut *tx) + .await?; - sqlx::query!( - "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2)", - w_id, - &linked_var_paths + for (path, res_data) in &deleted { + // Every cascaded variable this resource's value points at, ownership aside: restoring + // it on its own must bring back each secret it needs, and the one it borrowed from a + // sibling in the same batch is gone too. + let mut refs: Vec = Vec::new(); + if let Some(value) = res_data.get("value") { + collect_var_refs(value, &mut refs); + } + let this_linked: Vec = trash_linked_vars + .iter() + .filter(|(var_path, _)| { + refs.contains(var_path) && deleted_linked_variables.contains(var_path) + }) + .map(|(_, row)| row.clone()) + .collect(); + + let mut trash_data = serde_json::json!({"row": res_data}); + if !this_linked.is_empty() { + trash_data["linked_variables"] = serde_json::Value::Array(this_linked); + } + windmill_common::trashbin::move_to_trash( + &mut *tx, + &w_id, + "resource", + path, + trash_data, + &authed.username, ) - .execute(&mut *tx) .await?; } @@ -1746,13 +1929,30 @@ async fn delete_resources_bulk( ) .await?; + // See delete_resource: a cascaded variable gets the audit row it would have had. + for var_path in &deleted_linked_variables { + let params = cascade + .owner_of(var_path, &deleted_paths) + .map(|resource_path| HashMap::from([("via_resource", resource_path)])); + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(var_path), + params, + ) + .await?; + } + tx.commit().await?; // Wipe ALL users' drafts at these paths (and linked variables); see delete_resource. for path in &deleted_paths { delete_all_drafts_for_path(&db, &w_id, UserDraftItemKind::Resource, path).await?; } - for var_path in &linked_var_paths { + for var_path in &deleted_linked_variables { delete_all_drafts_for_path(&db, &w_id, UserDraftItemKind::Variable, var_path).await?; } @@ -2541,7 +2741,7 @@ async fn list_resource_types( ) -> JsonResult> { let rows = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ BY name", &w_id ) @@ -2566,6 +2766,352 @@ async fn list_resource_types_names( Ok(Json(rows)) } +#[derive(Serialize)] +struct ResourceTypeCount { + resource_type: String, + count: i64, +} + +/// How many resources of each type this workspace holds — how popular a type is *here*, +/// which is what the pickers rank on below the hub's own pick counts. +async fn list_resource_counts_by_type( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + // A count per type is aggregate, so there is no path to narrow it by: a token scoped + // to individual resources gets nothing rather than a total spanning paths it cannot + // read. Callers treat the refusal as "no local signal". + check_scopes(&authed, || "resources:read".to_string())?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| ResourceTypeCount { resource_type: r.resource_type, count: r.count }) + .collect(), + )) +} + +/// A hub read, remembered with the hub it came from: `hub_base_url` is a live instance +/// setting, so a cache ignoring it would keep serving the previous hub's answers. +struct HubCached { + hub_base_url: String, + fetched_at: std::time::Instant, + value: T, +} + +/// The index changes only when a resource type is published to the hub; picks move slowly +/// and only reorder a list. Both are read on every drawer open, hence caching at all. +const HUB_RT_INDEX_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60); +const HUB_RT_PICKS_TTL: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// How long a *failed* index read is remembered. Short, and deliberately not the hour a +/// success is good for: this read is on the path of every picker open, so an unreachable +/// hub must not cost an outbound timeout each time, while one blip must not silence pick +/// reporting for an hour. +const HUB_RT_INDEX_FAILURE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +static HUB_RT_INDEX: LazyLock< + std::sync::RwLock>>>>, +> = LazyLock::new(|| std::sync::RwLock::new(None)); +static HUB_RT_PICKS: LazyLock>>>> = + LazyLock::new(|| std::sync::RwLock::new(None)); + +fn hub_cache_get( + cache: &std::sync::RwLock>>, + hub_base_url: &str, + ttl: std::time::Duration, +) -> Option { + let guard = cache.read().ok()?; + let entry = guard.as_ref()?; + (entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl) + .then(|| entry.value.clone()) +} + +fn hub_cache_put(cache: &std::sync::RwLock>>, hub_base_url: &str, value: T) { + if let Ok(mut guard) = cache.write() { + *guard = Some(HubCached { + hub_base_url: hub_base_url.to_string(), + fetched_at: std::time::Instant::now(), + value, + }); + } +} + +#[derive(Deserialize)] +struct HubResourceTypeEntry { + id: i64, + name: String, + /// Optional so a hub that does not send it costs only the mapping. Required, it would + /// fail the whole parse and take pick reporting — which needs just the id — with it. + #[serde(default)] + app: Option, +} + +#[derive(Clone)] +struct HubResourceType { + id: i64, + /// The integration the type belongs to. Usually the type's own name, but not always: + /// `discord_webhook` and `discord_bot_configuration` are both `discord`, and only the + /// hub knows that. Without it a workspace holding a `discord_webhook` resource looks + /// like one that has never touched Discord. + app: String, +} + +/// Reads the index cache, choosing the TTL by what is stored: a failure expires far sooner +/// than a success. `None` is a miss, `Some(None)` a remembered failure. +fn hub_index_cached(hub_base_url: &str) -> Option>> { + let guard = HUB_RT_INDEX.read().ok()?; + let entry = guard.as_ref()?; + if entry.hub_base_url != hub_base_url { + return None; + } + let ttl = if entry.value.is_some() { + HUB_RT_INDEX_TTL + } else { + HUB_RT_INDEX_FAILURE_TTL + }; + (entry.fetched_at.elapsed() < ttl).then(|| entry.value.clone()) +} + +/// What the hub knows about every published resource type, keyed by the name Windmill +/// addresses it by. `None` when the hub cannot be reached or does not answer with a list. +async fn hub_resource_types( + db: &DB, + hub_base_url: &str, +) -> Option> { + if let Some(cached) = hub_index_cached(hub_base_url) { + return cached; + } + let index = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/list"), + false, + None, + Some(db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + // Only the id and the app are kept. That listing carries every type's schema — + // around a megabyte — and neither reporting a pick nor grouping types by + // integration needs it. + Some( + response + .json::>() + .await + .ok()? + .into_iter() + .map(|rt| { + let app = rt.app.unwrap_or_else(|| rt.name.clone()); + (rt.name, HubResourceType { id: rt.id, app }) + }) + .collect::>(), + ) + } + .await; + + hub_cache_put(&HUB_RT_INDEX, hub_base_url, index.clone()); + index +} + +#[derive(Serialize)] +struct PickHubResourceTypeResult { + success: bool, +} + +/// Tells the hub a resource type was taken into a workspace, which is the counter its +/// `/resource_types/picked` ranking reads. +/// +/// Never fails the caller: a hub predating the route, an unreachable one, and a type that +/// is local-only all mean the same thing — not counted — and the request that reaches here +/// has already saved the user's resource. +/// +/// POST, and scoped as a write, because it changes state on the hub under the instance's +/// own credentials. The sibling `/type/*` routes are metadata reads that a `resources:run` +/// app-embed token may make, and both the method and this check keep such a token — which +/// is untrusted app JavaScript — from driving hub counters through us. +async fn pick_hub_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Path((_w_id, name)): Path<(String, String)>, +) -> JsonResult { + check_scopes(&authed, || "resources:write".to_string())?; + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let success = async { + let id = hub_resource_types(&db, &hub_base_url).await?.get(&name)?.id; + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/{id}/pick"), + false, + None, + Some(&db), + ) + .await + .ok()?; + Some(response.status().is_success()) + } + .await + .unwrap_or(false); + + if !success { + tracing::debug!("hub did not record a pick for resource type {name}"); + } + Ok(Json(PickHubResourceTypeResult { success })) +} + +#[derive(Deserialize, Clone)] +struct HubResourceTypePicks { + name: String, + /// The hub counts picks in a bigint, which its driver serialises as a string. + #[serde(deserialize_with = "windmill_common::more_serde::maybe_number")] + picks: i64, +} + +#[derive(Deserialize)] +struct HubPickedResourceTypes { + resource_types: Vec, +} + +#[cfg(test)] +mod hub_picks_tests { + use super::*; + + /// Two properties of the index cache that a later edit could quietly drop: it is keyed on + /// the hub it came from, and a failure is forgotten long before a success is. + #[test] + fn the_index_cache_is_keyed_on_the_hub_and_forgets_failures_sooner() { + let index = || { + Some(HashMap::from([( + "slack".to_string(), + HubResourceType { id: 1, app: "slack".to_string() }, + )])) + }; + + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", index()); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_some())); + // Switching hubs must miss rather than serve the previous hub's mapping. + assert!(hub_index_cached("https://other.example").is_none()); + + // A remembered failure reads as a hit (so the hub is not re-attempted) carrying + // nothing, and only until the shorter of the two TTLs. + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", None); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_none())); + assert!(HUB_RT_INDEX_FAILURE_TTL < HUB_RT_INDEX_TTL); + + if let Ok(mut guard) = HUB_RT_INDEX.write() { + *guard = None; + } + } + + /// The hub counts picks in a bigint, which postgres.js serialises as a string. Typing + /// the field as a plain i64 fails the whole response, and the ranking silently empties. + #[test] + fn picks_decode_from_a_string_or_a_number() { + let parsed: HubPickedResourceTypes = serde_json::from_str( + r#"{"resource_types":[{"name":"slack","picks":"42"},{"name":"github","picks":7}]}"#, + ) + .unwrap(); + assert_eq!( + parsed + .resource_types + .iter() + .map(|rt| (rt.name.as_str(), rt.picks)) + .collect::>(), + vec![("slack", 42), ("github", 7)] + ); + } +} + +/// One hub resource type as the pickers need it. +#[derive(Serialize)] +struct HubResourceTypeInfo { + name: String, + /// The integration it belongs to, so a caller can total a workspace's resources per + /// integration rather than per type. + app: String, + picks: i64, +} + +/// What the hub knows about its resource types: which integration each belongs to, and how +/// often each has been picked. +/// +/// Empty rather than an error when the hub answers neither read, so the pickers treat an +/// older or private hub as "no hub signal" and fall back to what the workspace itself uses. +/// The two reads degrade independently: a hub that lists types but has no `picked` route +/// still supplies the type-to-integration mapping, which is what decides whether a +/// workspace's resources are recognised as belonging to an integration at all. +async fn list_hub_resource_type_info( + Extension(db): Extension, +) -> JsonResult> { + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let picks = match hub_cache_get(&HUB_RT_PICKS, &hub_base_url, HUB_RT_PICKS_TTL) { + Some(picks) => picks, + None => { + let fetched = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/picked"), + false, + Some(vec![("limit", "200".to_string())]), + Some(&db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + Some( + response + .json::() + .await + .ok()? + .resource_types, + ) + } + .await + .unwrap_or_default(); + hub_cache_put(&HUB_RT_PICKS, &hub_base_url, fetched.clone()); + fetched + } + }; + + let mut picks_by_name: HashMap = + picks.into_iter().map(|rt| (rt.name, rt.picks)).collect(); + let index = hub_resource_types(&db, &hub_base_url) + .await + .unwrap_or_default(); + + let mut info: Vec = index + .into_iter() + .map(|(name, rt)| HubResourceTypeInfo { + picks: picks_by_name.remove(&name).unwrap_or(0), + name, + app: rt.app, + }) + .collect(); + // What the index did not account for is a type the picks read knows and the listing does + // not — which is what a hub answering only the second read looks like. Its own name is + // the same guess a caller makes for any type the mapping misses. + info.extend( + picks_by_name + .into_iter() + .map(|(name, picks)| HubResourceTypeInfo { app: name.clone(), name, picks }), + ); + + Ok(Json(info)) +} + async fn get_resource_type( authed: ApiAuthed, Extension(user_db): Extension, @@ -2575,7 +3121,7 @@ async fn get_resource_type( let resource_type_o = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", &name, &w_id ) @@ -2603,6 +3149,20 @@ async fn exists_resource_type( Ok(Json(exists)) } +/// Trimmed, blank as none, and held to the column's 100 characters, so an over-long name is +/// refused with a message rather than a database error. +fn normalize_display_name(name: Option<&str>) -> Result> { + let Some(name) = name.map(str::trim).filter(|n| !n.is_empty()) else { + return Ok(None); + }; + if name.chars().count() > 100 { + return Err(Error::BadRequest( + "display_name must be at most 100 characters".to_string(), + )); + } + Ok(Some(name.to_string())) +} + async fn create_resource_type( authed: ApiAuthed, Extension(db): Extension, @@ -2636,11 +3196,12 @@ async fn create_resource_type( "A fileset resource type cannot have a format_extension".to_string(), )); } + let display_name = normalize_display_name(resource_type.display_name.as_deref())?; sqlx::query!( "INSERT INTO resource_type - (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", w_id, resource_type.name, resource_type.schema, @@ -2648,6 +3209,7 @@ async fn create_resource_type( authed.username, resource_type.format_extension, is_fileset, + display_name, ) .execute(&mut *tx) .await?; @@ -2815,6 +3377,12 @@ async fn update_resource_type( None => sqlb.set("format_extension", "NULL"), }; } + if let Some(display_name) = &ns.display_name { + match normalize_display_name(display_name.as_deref())? { + Some(name) => sqlb.set_str("display_name", name), + None => sqlb.set("display_name", "NULL"), + }; + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; @@ -3065,8 +3633,10 @@ fn git_url_userinfo(url: &str) -> Option<&str> { git_url_userinfo_range(url).map(|r| &url[r]) } -/// Validates a git URL to prevent option injection, SSRF, and local file read. -async fn validate_git_url(url: &str) -> Result<()> { +/// Validates a git URL to prevent option injection, SSRF, and local file read. The +/// syntax and scheme checks apply to every caller; the private-host refusal only +/// where [`private_git_host_allowed`] refuses `caller`. +async fn validate_git_url(url: &str, caller: GitRemoteCaller) -> Result<()> { let url = url.trim(); if url.is_empty() { return Err(Error::BadRequest("Git URL cannot be empty".to_string())); @@ -3120,25 +3690,26 @@ async fn validate_git_url(url: &str) -> Result<()> { let host = extract_host_from_git_url(url) .ok_or_else(|| Error::BadRequest("Could not parse hostname from git URL".to_string()))?; - // CI/dev escape hatch: integration tests run their git remote (a Gitea - // container) on localhost, which the network-target checks below reject. - // Scheme and option-injection validation above still applies. - if std::env::var("ALLOW_LOCAL_GIT_REMOTES").is_ok_and(|v| v == "true" || v == "1") { + // Scheme and option-injection validation above applies to every caller. + if private_git_host_allowed(caller) { return Ok(()); } + let hint = private_git_host_hint(caller) + .map(|h| format!(" {h}")) + .unwrap_or_default(); if host == "localhost" || host.ends_with(".local") || host == "[::1]" { - return Err(Error::BadRequest( - "Git URLs targeting localhost or local network are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting localhost or local network are not allowed.{hint}" + ))); } // Check literal IP addresses if let Ok(ip) = host.parse::() { if is_private_or_reserved_ip(&ip) { - return Err(Error::BadRequest( - "Git URLs targeting private or reserved IP addresses are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting private or reserved IP addresses are not allowed.{hint}" + ))); } } else { // Hostname — resolve via DNS and reject if any address is private. Fail @@ -3159,9 +3730,9 @@ async fn validate_git_url(url: &str) -> Result<()> { } for addr in addrs { if is_private_or_reserved_ip(&addr.ip()) { - return Err(Error::BadRequest( - "Git URL hostname resolves to a private or reserved IP address".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URL hostname resolves to a private or reserved IP address.{hint}" + ))); } } } @@ -3263,8 +3834,24 @@ async fn get_git_commit_hash( .map_err(|e| { Error::BadRequest(format!("Invalid git repository resource format: {}", e)) })?; + let caller = if authed.is_admin { + GitRemoteCaller::AdminOrSystem + } else { + GitRemoteCaller::NonAdmin + }; git_resource.url = - resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false).await?; + resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false, caller) + .await?; + // A credential is stored under the repository it was issued for, so a + // resource repointed elsewhere finds none. Which credential can be attached + // is bounded by that; who may use it is bounded here, on the same terms as + // the installation credential above. + let plain_url = git_resource.url.clone(); + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(&db, &w_id, git_resource.url).await?; + if git_resource.url != plain_url { + require_admin(authed.is_admin, &authed.username)?; + } let identities: Vec = query .git_ssh_identity @@ -3284,7 +3871,7 @@ async fn get_git_commit_hash( let (git_ssh_cmd, filenames) = get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; - let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd, caller).await; delete_paths(&filenames).await; @@ -3388,12 +3975,17 @@ async fn get_git_ssh_cmd( const GIT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// `git` command for a remote probe, with HTTP redirects disabled. `validate_git_url` -/// only vets the host in the URL; git's default (`http.followRedirects=initial`) -/// would let a validated public remote 302 the probe onto a private or link-local -/// address that no check ever sees. Build every probe through this. +/// checks the host in the URL, never one a redirect names; git's default +/// (`http.followRedirects=initial`) would let a public remote 302 the probe of a +/// caller refused private hosts onto one. Build every probe through this. +/// +/// The transports are pinned too: an SCP-shaped remote-helper string such as +/// `ext::@host:path` passes the URL check for a caller allowed private +/// hosts, and only git's own config would stop it from running the command. fn git_probe_command() -> Command { let mut git_cmd = Command::new("git"); git_cmd.args(["-c", "http.followRedirects=false"]); + git_cmd.env("GIT_ALLOW_PROTOCOL", "http:https:ssh:git"); git_cmd } @@ -3453,8 +4045,8 @@ fn dot_git_url(url: &str) -> Option { } /// Run a remote probe, retrying against [`dot_git_url`] if the remote answered the -/// URL as given with a redirect. Extending the path keeps the retry on the host -/// `validate_git_url` already cleared, which is exactly what following the redirect +/// URL as given with a redirect. Extending the path keeps the retry on the host of +/// the URL `validate_git_url` checked, which is exactly what following the redirect /// would not guarantee. `build` must produce the probe for the URL it is handed. /// /// A retry that also fails reports the *original* failure, so the caller's message @@ -3594,6 +4186,7 @@ async fn resolve_azure_devops_url( w_id: &str, url: &str, allow_cache: bool, + caller: GitRemoteCaller, ) -> Result { // Trim first: the http(s) gates the callers apply trim too, so a stored URL with // leading whitespace must not reach the scheme check here as a non-http one. @@ -3606,7 +4199,7 @@ async fn resolve_azure_devops_url( // cost a live credential (nor cache one), and whoever can edit the URL would // otherwise drive a token mint per poll tick. let probe_url = url.replace(placeholder, "windmill"); - validate_git_url(&probe_url).await?; + validate_git_url(&probe_url, caller).await?; // The background poller reads the referenced resource under the system identity, // which bypasses RLS. Confining the destination is what keeps that from becoming an @@ -3806,9 +4399,10 @@ fn git_sync_system_dba(db: &DB) -> DbWithOptAuthed<'static, ApiAuthed> { async fn get_repo_latest_commit_hash( git_resource: &GitRepositoryResource, git_ssh_command: Option, + caller: GitRemoteCaller, ) -> Result { // Validate URL and branch to prevent option injection and SSRF attacks - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, caller).await?; let ref_spec = git_resource .branch @@ -3941,19 +4535,30 @@ pub async fn get_git_repo_head_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } + git_resource.url = resolve_azure_devops_url( + &git_sync_system_dba(db), + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; + // A repo whose credential Windmill holds carries none in its URL, so the + // poller has to attach it here or every probe would be unauthenticated. git_resource.url = - resolve_azure_devops_url(&git_sync_system_dba(db), w_id, &git_resource.url, true).await?; + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; if let Some(branch) = git_resource.branch.as_deref().filter(|s| !s.is_empty()) { let branch = branch.to_string(); - let sha = get_repo_latest_commit_hash(&git_resource, None).await?; + let sha = get_repo_latest_commit_hash(&git_resource, None, GitRemoteCaller::AdminOrSystem) + .await?; return Ok(Some((branch, sha))); } // No explicit branch: resolve the remote's default-branch NAME along with // its head in one call. Fork sync needs the concrete name to scope // `wm-fork//*`, so a bare "HEAD" ref would silently disable it. - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; let output = run_git_probe_for_url(&git_resource.url, "ls-remote --symref HEAD", |url| { let mut git_cmd = git_probe_command(); git_cmd.args(["ls-remote", "--symref", url, "HEAD"]); @@ -4048,8 +4653,20 @@ pub async fn get_git_repo_fork_heads_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } - git_resource.url = resolve_azure_devops_url(&dba, w_id, &git_resource.url, true).await?; - validate_git_url(&git_resource.url).await?; + git_resource.url = resolve_azure_devops_url( + &dba, + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; + // Same reason as the head probe above: a repository whose credential Windmill + // holds carries none in its URL, and listing the fork branches is the half of + // polling that would otherwise go out unauthenticated. + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; validate_git_ref(base_branch)?; for r in extra_refs { @@ -4435,46 +5052,52 @@ mod tests { )); } + // A caller let through to private hosts must still hit the scheme check. #[tokio::test] async fn test_validate_git_url_blocks_file_scheme() { - let result = validate_git_url("file:///etc/passwd").await; + let result = validate_git_url("file:///etc/passwd", GitRemoteCaller::AdminOrSystem).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("https://")); } #[tokio::test] async fn test_validate_git_url_blocks_private_ips() { - assert!(validate_git_url("http://127.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://169.254.169.254/latest/meta-data/") - .await - .is_err()); - assert!(validate_git_url("http://10.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://172.16.0.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("http://192.168.1.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("git://0.0.0.0/repo.git").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1/repo.git").await.is_err()); + assert!(v("http://169.254.169.254/latest/meta-data/").await.is_err()); + let err = v("http://10.0.0.1/repo.git").await.unwrap_err(); + assert!(err.to_string().contains("ALLOW_LOCAL_GIT_REMOTES"), "{err}"); + assert!(v("http://172.16.0.1/repo.git").await.is_err()); + assert!(v("http://192.168.1.1/repo.git").await.is_err()); + assert!(v("git://0.0.0.0/repo.git").await.is_err()); // IPv6 loopback, unique-local, and link-local literals - assert!(validate_git_url("git://[::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fd00::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fe80::1]/repo.git").await.is_err()); + assert!(v("git://[::1]/repo.git").await.is_err()); + assert!(v("git://[fd00::1]/repo.git").await.is_err()); + assert!(v("git://[fe80::1]/repo.git").await.is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_lets_admins_reach_private_hosts() { + assert!( + validate_git_url("http://10.0.0.1/repo.git", GitRemoteCaller::AdminOrSystem) + .await + .is_ok() + ); } #[tokio::test] async fn test_validate_git_url_blocks_localhost() { - assert!(validate_git_url("http://localhost/repo.git").await.is_err()); - assert!(validate_git_url("http://myhost.local/repo.git") - .await - .is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://localhost/repo.git").await.is_err()); + assert!(v("http://myhost.local/repo.git").await.is_err()); } #[tokio::test] async fn test_validate_git_url_blocks_local_paths() { - assert!(validate_git_url("/etc/passwd").await.is_err()); - assert!(validate_git_url("../relative/path").await.is_err()); - assert!(validate_git_url("./local/repo").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("/etc/passwd").await.is_err()); + assert!(v("../relative/path").await.is_err()); + assert!(v("./local/repo").await.is_err()); } /// Minimal loopback HTTP server: replies to every request with `response` and @@ -4604,7 +5227,11 @@ mod tests { async fn test_validate_git_url_fails_closed_on_unresolvable_host() { // `.invalid` never resolves (RFC 6761). The private-IP check is only // meaningful if a failed lookup rejects instead of falling through. - let result = validate_git_url("https://this-host-does-not-exist.invalid/repo.git").await; + let result = validate_git_url( + "https://this-host-does-not-exist.invalid/repo.git", + GitRemoteCaller::NonAdmin, + ) + .await; assert!( result.is_err(), "an unresolvable host was allowed — does this resolver synthesize records for NXDOMAIN?" @@ -4615,21 +5242,33 @@ mod tests { #[tokio::test] async fn test_validate_git_url_allows_valid_urls() { // Needs DNS: validation fails closed on a host it cannot resolve. - assert!(validate_git_url("https://github.com/user/repo.git") + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("https://github.com/user/repo.git").await.is_ok()); + assert!(v("git@github.com:user/repo.git").await.is_ok()); + assert!(v("ssh://git@github.com/user/repo.git").await.is_ok()); + } + + #[tokio::test] + async fn test_git_probe_refuses_remote_helpers() { + // A caller allowed private hosts skips the DNS step that would reject this + // SCP-shaped string, so the transport pin is what keeps git from running it. + let output = git_probe_command() + .args(["ls-remote", "testhelper::x@127.0.0.1:repo"]) + .output() .await - .is_ok()); - assert!(validate_git_url("git@github.com:user/repo.git") - .await - .is_ok()); - assert!(validate_git_url("ssh://git@github.com/user/repo.git") - .await - .is_ok()); + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("transport 'testhelper' not allowed"), + "{stderr}" + ); } #[tokio::test] async fn test_validate_git_url_blocks_option_injection() { - assert!(validate_git_url("-evil").await.is_err()); - assert!(validate_git_url("--upload-pack=evil").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("-evil").await.is_err()); + assert!(v("--upload-pack=evil").await.is_err()); } #[test] @@ -4725,24 +5364,21 @@ mod tests { // GHSA-p5cj-8cfh-mjv6: a loopback authority must stay blocked, and the // fragment/query `@public-host` bypasses of #8600 must be rejected so the // host git dials can never diverge from the validated host. - assert!(validate_git_url("http://127.0.0.1:40173/repo.git") - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - // A legitimate public repo URL still validates. + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1:40173/repo.git").await.is_err()); assert!( - validate_git_url("https://github.com/windmill-labs/windmill.git") + v("http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git") .await - .is_ok() + .is_err() ); + assert!( + v("http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git") + .await + .is_err() + ); + // A legitimate public repo URL still validates. + assert!(v("https://github.com/windmill-labs/windmill.git") + .await + .is_ok()); } } diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 3766a6050d..773616a807 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -158,6 +158,7 @@ pub struct RunJob { pub payload: JobPayload, pub args: serde_json::Map, pub scheduled_for_o: Option>, + pub username: String, pub email: String, pub job_id: Option, pub workspace_id: String, @@ -169,6 +170,7 @@ impl From for RunJob { payload, args: Default::default(), scheduled_for_o: None, + username: "test-user".to_string(), email: "test@windmill.dev".to_string(), job_id: None, workspace_id: "test-workspace".to_string(), @@ -190,7 +192,11 @@ impl RunJob { self } - pub fn email(mut self, email: impl Into) -> Self { + /// Run as this workspace member. Both halves together, because the job's identity is the + /// principal: an address paired with another member's username is re-resolved at push to + /// the address that username holds. + pub fn as_user(mut self, username: impl Into, email: impl Into) -> Self { + self.username = username.into(); self.email = email.into(); self } @@ -206,7 +212,7 @@ impl RunJob { } pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args, scheduled_for_o, email, job_id, workspace_id } = self; + let RunJob { payload, args, scheduled_for_o, username, email, job_id, workspace_id } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -219,9 +225,9 @@ impl RunJob { &workspace_id, payload, windmill_queue::PushArgs::from(&hm_args), - /* user */ "test-user", + /* user */ &username, /* email */ &email, - /* permissioned_as */ "u/test-user".to_string(), + /* permissioned_as */ format!("u/{username}"), /* token_prefix */ None, /* audit_end_user */ None, scheduled_for_o, diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 68986bb6f9..01ddfd4a81 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -9,7 +9,7 @@ use sqlx::PgConnection; use std::collections::HashSet; use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::global_settings::HTTP_ROUTE_WORKSPACED_ROUTE; +use windmill_common::global_settings::{validate_allowed_origins, HTTP_ROUTE_WORKSPACED_ROUTE}; use windmill_common::{ db::UserDB, error::{Error, Result}, @@ -189,6 +189,7 @@ pub async fn insert_new_trigger_into_db( authentication_resource_path, wrap_body, raw_string, + allowed_origins, script_path, summary, description, @@ -207,7 +208,7 @@ pub async fn insert_new_trigger_into_db( retry ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, now(), $21, $22, $23, $24 ) "#, w_id, @@ -218,6 +219,7 @@ pub async fn insert_new_trigger_into_db( trigger.config.authentication_resource_path, trigger.config.wrap_body.unwrap_or(false), trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.base.script_path, trigger.config.summary, trigger.config.description, @@ -444,6 +446,7 @@ impl TriggerCrud for HttpTrigger { "workspaced_route", "wrap_body", "raw_string", + "allowed_origins", ]; fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject { @@ -474,6 +477,8 @@ impl TriggerCrud for HttpTrigger { validate_authentication_method(new.authentication_method, new.raw_string)?; + validate_allowed_origins(new.allowed_origins.as_deref().unwrap_or_default())?; + Ok(()) } @@ -492,6 +497,8 @@ impl TriggerCrud for HttpTrigger { validate_authentication_method(edit.authentication_method, edit.raw_string)?; + validate_allowed_origins(edit.allowed_origins.as_deref().unwrap_or_default())?; + Ok(()) } @@ -554,33 +561,35 @@ impl TriggerCrud for HttpTrigger { workspaced_route = $3, wrap_body = $4, raw_string = $5, - authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - mode = $10, - http_method = $11, - static_asset_config = $12, - edited_by = $13, - permissioned_as = $14, - request_type = $15, - authentication_method = $16, - summary = $17, - description = $18, + allowed_origins = $6, + authentication_resource_path = $7, + script_path = $8, + path = $9, + is_flow = $10, + mode = $11, + http_method = $12, + static_asset_config = $13, + edited_by = $14, + permissioned_as = $15, + request_type = $16, + authentication_method = $17, + summary = $18, + description = $19, edited_at = now(), - is_static_website = $19, - error_handler_path = $20, - error_handler_args = $21, - retry = $22 + is_static_website = $20, + error_handler_path = $21, + error_handler_args = $22, + retry = $23 WHERE - workspace_id = $23 AND - path = $24 + workspace_id = $24 AND + path = $25 "#, route_path, &route_path_key, Some(effective_workspaced), trigger.config.wrap_body.unwrap_or(false), trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, @@ -613,30 +622,32 @@ impl TriggerCrud for HttpTrigger { SET wrap_body = $1, raw_string = $2, - authentication_resource_path = $3, - script_path = $4, - path = $5, - is_flow = $6, - mode = $7, - http_method = $8, - static_asset_config = $9, - edited_by = $10, - permissioned_as = $11, - request_type = $12, - authentication_method = $13, - summary = $14, - description = $15, + allowed_origins = $3, + authentication_resource_path = $4, + script_path = $5, + path = $6, + is_flow = $7, + mode = $8, + http_method = $9, + static_asset_config = $10, + edited_by = $11, + permissioned_as = $12, + request_type = $13, + authentication_method = $14, + summary = $15, + description = $16, edited_at = now(), - is_static_website = $16, - error_handler_path = $17, - error_handler_args = $18, - retry = $19 + is_static_website = $17, + error_handler_path = $18, + error_handler_args = $19, + retry = $20 WHERE - workspace_id = $20 AND - path = $21 + workspace_id = $21 AND + path = $22 "#, trigger.config.wrap_body.unwrap_or(false), trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs index b78276c28d..40bab102a5 100644 --- a/backend/windmill-trigger-http/src/lib.rs +++ b/backend/windmill-trigger-http/src/lib.rs @@ -8,7 +8,7 @@ use tokio::sync::{RwLock, RwLockReadGuard}; use windmill_common::{ error::{Error, Result}, flows::Retry, - global_settings::HTTP_ROUTE_WORKSPACED_ROUTE, + global_settings::{allows_any_origin, HTTP_ROUTE_WORKSPACED_ROUTE}, utils::ExpiringCacheEntry, worker::CLOUD_HOSTED, DB, @@ -51,6 +51,7 @@ pub struct TriggerRoute { pub workspaced_route: bool, pub wrap_body: bool, pub raw_string: bool, + pub allowed_origins: Option>, pub error_handler_path: Option, pub error_handler_args: Option>>, pub retry: Option>, @@ -127,6 +128,7 @@ pub struct HttpConfig { pub workspaced_route: bool, pub wrap_body: bool, pub raw_string: bool, + pub allowed_origins: Option>, } #[derive(Debug, Clone, Serialize)] @@ -144,6 +146,7 @@ pub struct HttpConfigRequest { pub workspaced_route: Option, pub wrap_body: Option, pub raw_string: Option, + pub allowed_origins: Option>, } #[derive(Deserialize)] @@ -162,6 +165,7 @@ struct HttpConfigRequestHelper { workspaced_route: Option, wrap_body: Option, raw_string: Option, + allowed_origins: Option>, } impl<'de> Deserialize<'de> for HttpConfigRequest { @@ -197,6 +201,7 @@ impl<'de> Deserialize<'de> for HttpConfigRequest { workspaced_route: helper.workspaced_route, wrap_body: helper.wrap_body, raw_string: helper.raw_string, + allowed_origins: helper.allowed_origins, }) } } @@ -216,6 +221,50 @@ pub struct RouteExists { pub workspaced_route: Option, } +/// The allowlist that governs a route: its own when it has one, otherwise the +/// instance-wide default. `None` means nothing is configured at either level, so +/// the route keeps the historical permissive behaviour. +/// +/// A list containing `*` is treated as no restriction, which is how a route opts +/// out of a stricter instance default. +pub fn effective_allowed_origins<'a>( + route_allowed_origins: Option<&'a [String]>, + instance_default: &'a [String], +) -> Option<&'a [String]> { + // An empty list is not a configuration. It reads exactly as never having set + // one, so such a route still inherits the instance default rather than + // skipping it, which is what would make `[]` more permissive than `NULL`. + match route_allowed_origins.filter(|list| !list.is_empty()) { + // `*` is the opt-out, including out of a stricter instance default. + Some(list) if allows_any_origin(list) => None, + Some(list) => Some(list), + None => (!instance_default.is_empty() && !allows_any_origin(instance_default)) + .then_some(instance_default), + } +} + +/// Resolve the `Access-Control-Allow-Origin` value for a request, or `None` to +/// omit the header so the browser blocks the read. +/// +/// The request's `Origin` is echoed back only on a match against the allowlist. +/// Reflecting it unchecked is the classic way this feature turns into no +/// restriction at all. +/// +/// The comparison ignores ASCII case because a browser lowercases the scheme and +/// host it sends, so a configured `https://App.Example.com` would otherwise name +/// a real origin and still match nothing. +pub fn match_origin( + allowed_origins: &[String], + origin: Option<&http::HeaderValue>, +) -> Option { + let origin = origin?; + let origin_str = origin.to_str().ok()?; + allowed_origins + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(origin_str)) + .then(|| origin.clone()) +} + pub fn validate_authentication_method( authentication_method: AuthenticationMethod, raw_string: Option, @@ -276,6 +325,7 @@ pub async fn refresh_routers( static_asset_config AS "static_asset_config: _", wrap_body, raw_string, + allowed_origins, workspaced_route, is_static_website, error_handler_path, @@ -384,6 +434,10 @@ pub struct HttpTrigger; #[cfg(test)] mod tests { use super::*; + // Not used by the lib itself, only exercised here. + use windmill_common::global_settings::{ + validate_allowed_origins, MAX_ALLOWED_ORIGINS, MAX_ALLOWED_ORIGIN_LEN, + }; #[test] fn test_request_type_backward_compatibility() { @@ -578,6 +632,168 @@ mod tests { assert!(validate_authentication_method(AuthenticationMethod::Signature, None).is_ok()); } + // --- CORS allowed origins --- + + fn origin(value: &str) -> http::HeaderValue { + http::HeaderValue::from_str(value).unwrap() + } + + #[test] + fn test_match_origin_exact_match_echoes_request_origin() { + let allowed = vec!["https://a.com".to_string(), "https://b.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://b.com"))), + Some(origin("https://b.com")) + ); + } + + #[test] + fn test_match_origin_ignores_case() { + let allowed = vec!["https://App.Example.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://app.example.com"))), + Some(origin("https://app.example.com")) + ); + } + + #[test] + fn test_match_origin_no_match_omits_header() { + let allowed = vec!["https://a.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://evil.com"))), + None + ); + // A prefix of an allowed origin must not match: https://a.com.evil.com + // is a different site entirely. + assert_eq!( + match_origin(&allowed, Some(&origin("https://a.com.evil.com"))), + None + ); + } + + #[test] + fn test_wildcard_entry_means_unrestricted() { + // `*` is handled before matching: it means "no restriction", which is + // how a route opts out of a stricter instance default. + assert!(allows_any_origin(&["*".to_string()])); + assert!(allows_any_origin(&[ + "https://a.com".to_string(), + "*".to_string() + ])); + assert!(!allows_any_origin(&["https://a.com".to_string()])); + assert_eq!( + effective_allowed_origins(Some(&["*".to_string()]), &[]), + None + ); + } + + #[test] + fn test_effective_allowed_origins_prefers_the_route() { + let route = ["https://a.com".to_string()]; + let default = ["https://default.com".to_string()]; + assert_eq!( + effective_allowed_origins(Some(&route), &default), + Some(&route[..]) + ); + // No route list: the instance default applies. + assert_eq!( + effective_allowed_origins(None, &default), + Some(&default[..]) + ); + // No route list and no instance default: nothing is restricted, so the + // historical permissive behaviour is kept. + assert_eq!(effective_allowed_origins(None, &[]), None); + // A route opting out with `*` escapes a stricter instance default. + assert_eq!( + effective_allowed_origins(Some(&["*".to_string()]), &default), + None + ); + // An empty route list is not a configuration: it resolves exactly as + // `NULL` does, so it inherits the instance default rather than skipping + // it and becoming more permissive than an unset one. + assert_eq!( + effective_allowed_origins(Some(&[]), &default), + Some(&default[..]) + ); + assert_eq!(effective_allowed_origins(Some(&[]), &[]), None); + } + + #[test] + fn test_match_origin_missing_origin_header_omits_header() { + let allowed = vec!["https://a.com".to_string()]; + assert_eq!(match_origin(&allowed, None), None); + } + + #[test] + fn test_validate_allowed_origins_accepts_anything_comparable() { + // A shape that cannot match simply matches nothing, so it is the + // editor's job to warn and not this one's to refuse. What is refused is + // narrower: `null`, values that are not header-comparable, entries that + // cannot round-trip the editor's comma-separated field, and lists past + // the size a request can afford to scan. + let allowed = vec![ + "https://app.example.com".to_string(), + "http://localhost:3000".to_string(), + "http://[::1]:8080".to_string(), + "chrome-extension://mhjfbmdgcfjbbpaeojofohoefgiehjai".to_string(), + // Never matches, but that is the caller's problem, not an error. + "https://app.example.com/".to_string(), + "https://app.example.com:99999".to_string(), + "not-an-origin".to_string(), + "*".to_string(), + ]; + assert!(validate_allowed_origins(&allowed).is_ok()); + assert!(validate_allowed_origins(&[]).is_ok()); + } + + #[test] + fn test_parse_allowed_origins_setting_rejects_empty_array_entries() { + use windmill_common::global_settings::parse_allowed_origins_setting; + // A trailing separator in the string form is a typing artifact and is + // dropped; an empty array entry is something the caller wrote, so it + // must reach validation rather than be filtered away into an empty + // (and therefore unrestricted) default. + assert!(parse_allowed_origins_setting(Some(&serde_json::json!("https://a.com,"))).is_ok()); + assert!(parse_allowed_origins_setting(Some(&serde_json::json!([""]))).is_err()); + assert!( + parse_allowed_origins_setting(Some(&serde_json::json!(["https://a.com", ""]))).is_err() + ); + } + + #[test] + fn test_validate_allowed_origins_bounds_the_list() { + // An allowlist is scanned on every request to a restricted route, the + // unauthenticated preflight included, so its size is a cost anyone can + // trigger. + let too_many = vec!["https://a.com".to_string(); MAX_ALLOWED_ORIGINS + 1]; + assert!(validate_allowed_origins(&too_many).is_err()); + assert!(validate_allowed_origins(&too_many[..MAX_ALLOWED_ORIGINS]).is_ok()); + let too_long = format!("https://{}.com", "a".repeat(MAX_ALLOWED_ORIGIN_LEN)); + assert!(validate_allowed_origins(&[too_long]).is_err()); + } + + #[test] + fn test_validate_allowed_origins_rejects_null_and_uncomparable() { + for invalid in [ + // Every sandboxed iframe sends `Origin: null`, so allowing it would + // grant access to any page that can open one. + "null", + "NULL", // Cannot be the string an Origin header is compared against. + "https://a b.com", + "https://app.example.com ", + "https://exämple.com", + // The editor edits the list as one comma-separated field, so an + // entry carrying a comma would come back as two and widen the list. + "https://a.com,https://b.com", + "", + ] { + assert!( + validate_allowed_origins(&[invalid.to_string()]).is_err(), + "expected {invalid} to be rejected" + ); + } + } + // --- Route path regex --- #[test] diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 8ef7c1ee39..66d6770354 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -18,8 +18,9 @@ use windmill_common::{ error::{Error, JsonResult, Result}, trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource}, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows, - overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, + WithDraftQuery, }, utils::{paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, @@ -96,14 +97,14 @@ pub trait TriggerCrud: Send + Sync + 'static { const DEPLOYMENT_NAME: &'static str; const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; const IS_ALLOWED_ON_CLOUD: bool; - /// Whether enabling this trigger in a fork while the parent has the same - /// path enabled is a real conflict (shared upstream resource). True for + /// Whether enabling this trigger in a fork while an ancestor workspace has + /// the same path is a real conflict (shared upstream resource). True for /// listener-based kinds where two consumers compete (Kafka group, PG slot, /// SQS queue, etc.) and for Websocket where both subscribers fire on every /// broadcast. False for kinds whose upstream identifier is implicitly /// workspace-scoped at runtime (HTTP routes, Email local_part — clones for /// the non-workspaced sub-case are filtered out, so any cloned row is - /// already collision-free vs. the parent). + /// already collision-free vs. its ancestors). const FORK_CONFLICT_ON_ENABLE: bool = true; fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject; @@ -990,6 +991,18 @@ async fn delete_trigger( .await?; if !deleted { + drop(tx); + if delete_draft_only_for_path( + &db, + &workspace_id, + T::user_draft_item_kind(), + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only trigger '{}' deleted", path)); + } return Err(Error::NotFound(format!( "Trigger not found at path: {}", path @@ -1082,54 +1095,14 @@ async fn exists_trigger( #[derive(serde::Deserialize)] struct SetTriggerModePayload { mode: TriggerMode, - /// When true, bypass the parent-state warning that would otherwise reject - /// enabling a trigger that's already enabled in the parent workspace. - /// The frontend sets this after the user confirms the duplicate-execution - /// dialog. See windmill-trigger/src/handler.rs::set_trigger_mode for the - /// full check. + /// When true, bypass the fork-conflict warning that would otherwise reject + /// enabling a trigger an ancestor workspace also has at this path. The + /// frontend sets this after the user confirms the duplicate-execution + /// dialog. See `set_trigger_mode` for the full check. #[serde(default)] force: bool, } -/// Returns the parent workspace id when this workspace is a fork *and* the -/// parent has a row at the same trigger path. Used to gate enabling a trigger -/// in a fork behind an explicit `force=true` confirmation: the fork's row was -/// cloned from the parent, so its upstream identifier (Kafka group, PG slot, -/// SQS queue URL, etc.) is shared by construction. The risk is independent of -/// the parent's current `mode`: if the parent is enabled, the two listeners -/// compete; if it's disabled, the fork can destructively take over shared -/// state (e.g. advance the PG WAL, claim an MQTT client_id) before the parent -/// re-enables. Either way, the user should be asked to confirm. -async fn parent_has_trigger( - tx: &mut PgConnection, - table_name: &str, - workspace_id: &str, - path: &str, -) -> Result> { - let parent: Option = - sqlx::query_scalar("SELECT parent_workspace_id FROM workspace WHERE id = $1") - .bind(workspace_id) - .fetch_optional(&mut *tx) - .await? - .flatten(); - let Some(parent_id) = parent else { - return Ok(None); - }; - let exists: Option = sqlx::query_scalar(&format!( - "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", - table_name - )) - .bind(&parent_id) - .bind(path) - .fetch_one(&mut *tx) - .await?; - Ok(if exists == Some(true) { - Some(parent_id) - } else { - None - }) -} - async fn set_trigger_mode( Extension(handler): Extension>, authed: ApiAuthed, @@ -1144,22 +1117,28 @@ async fn set_trigger_mode( let mut tx = user_db.begin(&authed).await?; // Block transitioning a trigger in a fork to any mode that attaches a - // listener (Enabled or Suspended) when the parent has the same path, + // listener (Enabled or Suspended) when an ancestor has the same path, // unless the caller passes force=true. Suspended still keeps the // listener attached — it just stops auto-running queued jobs — so a // suspended fork would still split Kafka events / share a PG slot - // with the parent. The cloned upstream identifier is shared by - // construction; the risk is independent of the parent's current mode. - // Skipped for kinds where the upstream identifier is already - // workspace-scoped at runtime (HTTP, Email). + // with the ancestor. The risk is independent of the ancestor's current + // mode: enabled, the two listeners compete; disabled, the fork can + // destructively take over shared state (advance the PG WAL, claim an + // MQTT client_id) before it re-enables. Skipped for kinds where the + // upstream identifier is already workspace-scoped at runtime (HTTP, Email). if T::FORK_CONFLICT_ON_ENABLE && payload.mode != TriggerMode::Disabled && !payload.force { - if let Some(parent_id) = - parent_has_trigger(&mut *tx, T::TABLE_NAME, &workspace_id, path).await? + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, + T::TABLE_NAME, + &workspace_id, + path, + ) + .await? { return Err(Error::BadRequest(format!( "fork-conflict:{}:{}", T::TRIGGER_TYPE, - parent_id + ancestor_id ))); } } diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index 463a365b06..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -14,17 +14,20 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, where `` is the name the - /// workspace configures it under. + /// A warehouse relation, `dbt:////`, where + /// `` is the name the workspace configures it under. /// - /// The SCHEME names the producer — dbt is the only thing that creates one — - /// while the PATH stays the physical relation, because that is what two - /// projects agree on: a mart one builds is a `source` the next reads, and - /// their dbt `unique_id`s differ (`model.a.orders` vs - /// `source.b.analytics.orders`) where the relation does not - /// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that - /// reader — the shared node is lineage, not a cascade edge. + /// The SCHEME names the namespace dbt made rather than an exclusive + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, + /// because that is what two producers agree on: a mart one builds is a + /// `source` the next reads, and their dbt `unique_id`s differ + /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does + /// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the + /// readers of what it built — that shared node is lineage, not a cascade + /// edge — while a declared write does (decision 25). Dbt, } diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 02feeb421a..cb1c53a579 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -538,6 +538,20 @@ pub struct Suspend { pub hide_cancel: Option, #[serde(skip_serializing_if = "false_or_empty")] pub continue_on_disapprove_timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skin: Option, +} + +/// How an approval request is presented, on the approval page and in Slack/Teams messages. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum ApprovalSkin { + Minimal, + /// A skin this server does not know renders as the detailed one rather than failing to + /// deserialize the whole flow, so a flow authored against a newer version still runs. + #[default] + #[serde(other)] + Detailed, } fn false_or_empty(v: &Option) -> bool { @@ -1365,6 +1379,23 @@ mod tests { assert_eq!(val.modules.len(), 1); } + #[test] + fn suspend_skin_unknown_value_falls_back_to_detailed() { + let skin_of = |skin: &str| { + let val: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": {"type": "identity"}, + "suspend": {"required_events": 1, "skin": skin} + }] + })) + .unwrap(); + val.modules[0].suspend.as_ref().unwrap().skin + }; + assert_eq!(skin_of("minimal"), Some(ApprovalSkin::Minimal)); + assert_eq!(skin_of("not_a_skin_yet"), Some(ApprovalSkin::Detailed)); + } + #[test] fn agent_tool_keeps_description_through_locking() { // #10244: the dependency job rebuilds each tool from its locked FlowModule; the diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 19f6a06021..9a059da6e8 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -612,6 +612,7 @@ impl Hash for NewScript { self.priority.hash(state); self.timeout.hash(state); self.delete_after_use.hash(state); + self.delete_after_secs.hash(state); self.restart_unless_cancelled.hash(state); self.deployment_message.hash(state); self.visible_to_runner_only.hash(state); diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs index a9003a3712..e2e4ab866b 100644 --- a/backend/windmill-worker-volumes/src/lib.rs +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -179,11 +179,24 @@ pub fn interpolate_volume_name( result } +/// PHP's opening tag, which is case-insensitive and may be followed by code. +fn is_php_open_tag(trimmed_line: &str) -> bool { + trimmed_line + .get(..5) + .is_some_and(|p| p.eq_ignore_ascii_case(" volume: ` lines, +/// stopping at the first line that is neither blank nor a comment. A PHP script +/// opens with ` Vec { let mut volumes = Vec::new(); for line in content.lines() { let trimmed = line.trim(); - if trimmed.is_empty() { + if trimmed.is_empty() || is_php_open_tag(trimmed) { continue; } if !trimmed.starts_with(comment_prefix) { @@ -230,6 +243,17 @@ mod tests { ); } + #[test] + fn parse_php_volume_after_open_tag() { + let content = + " { - const url = readFileSync(args.path, "utf8"); - const req = await fetch(url, { - method: "GET", - headers: { - Authorization: "Bearer " + token, - }, - }); + // A stalled fetch would otherwise hold the whole build for bun's own 5-minute + // default, with nothing naming the script it was waiting on. + const RELATIVE_IMPORT_FETCH_TIMEOUT_MS = 120000; + + function relativeImportFetchError(url, e) { + const reason = + e?.name === "TimeoutError" + ? `no response within ${RELATIVE_IMPORT_FETCH_TIMEOUT_MS / 1000}s` + : String(e?.message ?? e); + return new Error(`Failed to fetch relative import at ${url}: ${reason}`); + } + + async function fetchRelativeImport(url) { + let req; + try { + req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + signal: AbortSignal.timeout(RELATIVE_IMPORT_FETCH_TIMEOUT_MS), + }); + } catch (e) { + throw relativeImportFetchError(url, e); + } if (!req.ok) { throw new Error( - `Failed to find relative import at ${url}`, - req.statusText + `Failed to find relative import at ${url} (status ${req.status} ${req.statusText})` ); } - const contents = await req.text(); + try { + return await req.text(); + } catch (e) { + throw relativeImportFetchError(url, e); + } + } + + build.onLoad({ filter: /.*\.url$/ }, async (args) => { + const url = readFileSync(args.path, "utf8"); + const contents = await fetchRelativeImport(url); return { contents: replaceRelativeImports(contents).contents, loader: "tsx", @@ -105,9 +130,11 @@ const p = { const normalized = (isRelative ? join(dirname(file_path), pathNoExt) : pathNoExt.slice(1)).replace(/\\/g, "/"); const hash = TEMP_SCRIPT_REFS?.[normalized]; + // Lock generation substitutes `raw`: the dependency scan reads versions from the + // `pkg@version` specifiers in imported scripts, which `raw_unpinned` strips. const url = (isRelative - ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` - : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}` + ? `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${file_path}/../${args.path}${endExt}` + : `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${args.path}${endExt}` ) + (hash ? `?temp_script_hash=${hash}` : ""); const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js index a877266662..e9bb36c21e 100644 --- a/backend/windmill-worker/loader.bun.windows.js +++ b/backend/windmill-worker/loader.bun.windows.js @@ -101,6 +101,43 @@ const p = { return replaceRelativeImports(code); }); + // A stalled fetch would otherwise hold the whole build for bun's own 5-minute + // default, with nothing naming the script it was waiting on. + const RELATIVE_IMPORT_FETCH_TIMEOUT_MS = 120000; + + function relativeImportFetchError(url, e) { + const reason = + e?.name === "TimeoutError" + ? `no response within ${RELATIVE_IMPORT_FETCH_TIMEOUT_MS / 1000}s` + : String(e?.message ?? e); + return new Error(`Failed to fetch relative import at ${url}: ${reason}`); + } + + async function fetchRelativeImport(url) { + let req; + try { + req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + signal: AbortSignal.timeout(RELATIVE_IMPORT_FETCH_TIMEOUT_MS), + }); + } catch (e) { + throw relativeImportFetchError(url, e); + } + if (!req.ok) { + throw new Error( + `Failed to find relative import at ${url} (status ${req.status} ${req.statusText})` + ); + } + try { + return await req.text(); + } catch (e) { + throw relativeImportFetchError(url, e); + } + } + // Load windmill scripts by fetching from the API build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { // Extract temp_script_hash if embedded in the path by resolveWindmillImport @@ -110,18 +147,7 @@ const p = { : undefined; const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${scriptPath}` + (hashParam ? `?temp_script_hash=${hashParam}` : ""); - const req = await fetch(url, { - method: "GET", - headers: { - Authorization: "Bearer " + token, - }, - }); - if (!req.ok) { - throw new Error( - `Failed to find relative import at ${url} (status ${req.status})` - ); - } - const contents = await req.text(); + const contents = await fetchRelativeImport(url); return { contents: replaceRelativeImports(contents).contents, loader: "tsx", diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 6e80551650..51a00e65a7 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -26,11 +26,11 @@ use windmill_queue::{flow_status::get_step_of_flow_status, MiniPulledJob}; use crate::parse_sig_of_lang; -pub fn parse_raw_script_schema( +pub async fn parse_raw_script_schema( content: &str, language: &ScriptLang, ) -> Result, Error> { - let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)? + let main_arg_signature = parse_sig_of_lang(content, Some(&language), None).await? .ok_or_else(|| Error::BadConfig(format!( "Cannot parse signature for language {:?}. The language parser may not be enabled in this build.", language diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 2c29721ef6..156369e095 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -26,6 +26,10 @@ use windmill_ai::{ image_handler::upload_image_to_s3, providers::{ create_chat_completions_query_builder, create_query_builder, is_chat_completions_only, + openai::{ + is_reasoning_summary_unavailable, rejects_reasoning_summary, + remember_reasoning_summary_unavailable, + }, remember_chat_completions_only, }, proxy::{ @@ -49,7 +53,7 @@ use windmill_common::{ utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, Connection}, }; -use windmill_queue::{cancel_single_job, CanceledBy, MiniPulledJob}; +use windmill_queue::{append_logs, cancel_single_job, CanceledBy, MiniPulledJob}; use crate::{ ai::stream_event_processor::StreamEventProcessor, @@ -243,6 +247,88 @@ fn overlay_tool_inputs( } } +/// What every websearch entry is named by, whatever label it carries. Web search reaches the model +/// as a provider capability rather than a tool, so it has no model-facing name of its own, and the +/// editor's label is not one: a flow module tool could carry the same one, and enabling that tool +/// would then silently turn web search on with it. +/// +/// Reserved, on the `__wm_` prefix this codebase uses for names it keeps for itself, and held that +/// way by `flow_module_tool_name` refusing to advertise a tool that takes it. +const WEBSEARCH_ENABLED_NAME: &str = "__wm_web_search"; + +/// The name a flow module tool is advertised to the model under. +/// +/// Rejected rather than skipped: a tool the model is never shown is a tool the agent silently does +/// not have, and a run that quietly drops one is harder to explain than a run that will not start. +fn flow_module_tool_name(summary: Option<&str>) -> Result<&str, Error> { + match summary { + Some(name) if name == WEBSEARCH_ENABLED_NAME => Err(Error::internal_err(format!( + "Invalid tool name: {name:?} is reserved for enabling web search" + ))), + Some(name) if TOOL_NAME_REGEX.is_match(name) => Ok(name), + other => Err(Error::internal_err(format!("Invalid tool name: {other:?}"))), + } +} + +/// The name a run enables a roster entry by: the name the model is shown, except for an entry the +/// model is shown nothing of, which cannot be named by a label others may share. An MCP server is +/// named by the resource it points at, web search by `WEBSEARCH_ENABLED_NAME`. +/// +/// The MCP path is bare. The roster stores it as authored, `$res:` and all, but a name is an +/// argument value and one carrying that prefix is resolved to the resource itself before the worker +/// is handed its args, so the prefixed form is not something the list can hold. +fn tool_enabled_name(tool: &AgentTool) -> Option<&str> { + match &tool.value { + ToolValue::Mcp(mcp) => Some(mcp.resource_path.trim_start_matches("$res:")), + ToolValue::Websearch(_) => Some(WEBSEARCH_ENABLED_NAME), + _ => tool.summary.as_deref(), + } +} + +/// The roster a run advertises, given the names it enabled. +/// +/// `None` advertises the whole roster, which is what every agent written before the field existed +/// relies on; an empty list advertises nothing. +/// +/// Whole entries, decided before any of them is resolved: an MCP server a run switched off is never +/// contacted, and reading its resource, refreshing its token and opening a client are each things +/// that can fail a run. Which tools a server exposes stays its entry's own `include_tools` / +/// `exclude_tools`, the only place that choice is made. +fn narrow_roster(tools: Vec, enabled_tools: Option<&[String]>) -> Vec { + let Some(enabled) = enabled_tools else { + return tools; + }; + tools + .into_iter() + .filter(|t| tool_enabled_name(t).is_some_and(|name| enabled.iter().any(|n| n == name))) + .collect() +} + +/// The log line for names in `enabled_tools` that name nothing on the agent, if any. The list can +/// be computed per run, so a name that has since been renamed away must not fail the step — but it +/// would otherwise silently narrow the agent, so the run says how many of its names matched nothing. +/// +/// Counted, never quoted: a name is an argument value, and one written as `$var:path` reaches the +/// worker already replaced by the variable's own value. Quoting it would write that value to the +/// job log, where the masks a job registers never reach it — they are applied to a subprocess's +/// output in `handle_child` and nowhere else, and `append_logs` stores what it is given verbatim. +fn unmatched_enabled_tools_message( + enabled_tools: &[String], + advertised: &[&str], +) -> Option { + let unmatched = enabled_tools + .iter() + .filter(|name| !advertised.contains(&name.as_str())) + .count(); + if unmatched == 0 { + return None; + } + let subject = if unmatched == 1 { "name" } else { "names" }; + Some(format!( + "--- ENABLED TOOLS: {unmatched} {subject} named no tool of this agent and had no effect ---\n" + )) +} + pub async fn handle_ai_agent_job( // connection conn: &Connection, @@ -378,9 +464,10 @@ pub async fn handle_ai_agent_job( )); }; - // A linked step takes its brain and tools from the resource and keeps only the flow-local - // inputs (user_message/user_attachments) of its own; both stay rigid, so the one thing it may - // bind to this flow is the tools' inputs, overlaid from `tool_inputs` below. + // A linked step takes its brain and tools from the resource and keeps only its own flow-local + // inputs. The brain and the roster stay rigid; what the step binds to this flow is the message + // it asks, which of those tools this use may call, the conversation it is part of, and the + // tools' own inputs — the last overlaid from `tool_inputs` below. let (args, tools): (AIAgentArgs, Vec) = if let Some(agent_ref) = agent.as_deref() { let agent_path = agent_ref .trim_start_matches("$res:") @@ -434,7 +521,7 @@ pub async fn handle_ai_agent_job( // Only after interpolating the resource: these are caller-controlled and already resolved by // build_args_map, so passing them through it again would expand contextual values — // `$WM_TOKEN` in a user message would reach the model provider. - for key in ["user_message", "user_attachments"] { + for key in ["user_message", "user_attachments", "enabled_tools"] { if let Some(v) = local_args.get(key) { brain.insert( key.to_string(), @@ -477,6 +564,20 @@ pub async fn handle_ai_agent_job( tools }; + // Narrow the roster to the tools this run enabled, before the loop below pays a script or hub + // fetch per tool. + let enabled_tools = args.enabled_tools.as_deref(); + // Taken before the narrowing consumes the roster, and only by a run that narrows: they are what + // its names are matched against, so they are also what tells it a name matched nothing. + let roster_names: Vec = match enabled_tools { + Some(_) => tools + .iter() + .filter_map(|t| tool_enabled_name(t).map(str::to_string)) + .collect(), + None => Vec::new(), + }; + let tools = narrow_roster(tools, enabled_tools); + // Separate Windmill tools from MCP tools, websearch, and extract MCP resource configs let mut windmill_modules: Vec = Vec::new(); // Explicit per-tool descriptions keyed by tool id. When set, these override the @@ -541,12 +642,7 @@ pub async fn handle_ai_agent_job( let job = job; let user_description = tool_descriptions.get(&t.id).cloned(); async move { - let Some(summary) = t.summary.as_ref().filter(|s| TOOL_NAME_REGEX.is_match(s)) else { - return Err(Error::internal_err(format!( - "Invalid tool name: {:?}", - t.summary - ))); - }; + let summary = flow_module_tool_name(t.summary.as_deref())?; // Extract schema, input_transforms, and an auto-derived description from the module value let module_value = t.get_value()?; @@ -621,7 +717,7 @@ pub async fn handle_ai_agent_job( (schema, input_transforms, derived_description) } FlowModuleValue::RawScript { content, language, input_transforms, .. } => { - let schema = Some(parse_raw_script_schema(&content, &language)?); + let schema = Some(parse_raw_script_schema(&content, &language).await?); (schema, input_transforms, None) } FlowModuleValue::AIAgent { input_transforms, .. } => { @@ -657,7 +753,7 @@ pub async fn handle_ai_agent_job( def: ToolDef { r#type: "function".to_string(), function: ToolDefFunction { - name: summary.clone(), + name: summary.to_string(), description: Some(description), parameters: schema.unwrap_or_else(|| { to_raw_value(&serde_json::json!({ @@ -687,6 +783,22 @@ pub async fn handle_ai_agent_job( HashMap::new() }; + if let Some(enabled) = enabled_tools { + let matchable: Vec<&str> = roster_names.iter().map(|s| s.as_str()).collect(); + windmill_common::feature_usage::log_feature_usage( + "ai_agent", + "dynamic_tools", + if tools.is_empty() && !has_websearch { + "no_tools" + } else { + "tools" + }, + ); + if let Some(message) = unmatched_enabled_tools_message(enabled, &matchable) { + append_logs(&job.id, &job.workspace_id, message, conn).await; + } + } + let mut inner_occupancy_metrics = occupancy_metrics.clone(); let stream_notifier = StreamNotifier::new(conn, job); @@ -1103,6 +1215,7 @@ pub async fn run_agent( *has_stream = user_wants_streaming && is_text_output; let mut final_events_str = String::new(); + let mut final_reasoning = String::new(); // Always create a StreamEventProcessor for text output (use silent mode if user doesn't want streaming) let stream_event_processor = if is_text_output { @@ -1191,6 +1304,10 @@ pub async fn run_agent( attachments: args.user_attachments.as_deref(), has_websearch, prompt_cache_key: include_prompt_cache_key.then_some(prompt_cache_key.as_str()), + reasoning_summary: !is_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ), }; // A worker cannot run the client credentials exchange, so an OAuth resource @@ -1241,7 +1358,8 @@ pub async fn run_agent( // An endpoint can reject the request shape rather than the model: // `stream_options` and `prompt_cache_key`, which not every OpenAI-compatible - // gateway accepts, and the route itself, when an Azure resource is outside + // gateway accepts, a reasoning summary, which OpenAI refuses to unverified + // organizations, and the route itself, when an Azure resource is outside // the Responses API's model/region matrix. Each is retried once with that // part dropped. // Set where the route is found to be absent, and read once the fallback has @@ -1298,6 +1416,9 @@ pub async fn run_agent( && status.as_u16() == 400 && text.contains("prompt_cache_key"); + let summary_refused = build_args.reasoning_summary + && rejects_reasoning_summary(status.as_u16(), &text); + // Only the first call of the step may re-route: an endpoint that // does not serve this API rejects that one already, whereas a // rejection once the conversation is under way is about the @@ -1321,6 +1442,15 @@ pub async fn run_agent( ); include_prompt_cache_key = false; build_args.prompt_cache_key = None; + } else if summary_refused { + tracing::info!( + "Retrying request without the reasoning summary the endpoint refused" + ); + remember_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ); + build_args.reasoning_summary = false; } else if route_unserved { tracing::info!( "Endpoint rejected the request ({}), falling back to chat/completions", @@ -1355,6 +1485,7 @@ pub async fn run_agent( match parsed { ParsedResponse::Text { content: response_content, + reasoning: response_reasoning, tool_calls, events_str, annotations, @@ -1371,6 +1502,7 @@ pub async fn run_agent( if let Some(events_str) = events_str { final_events_str.push_str(&events_str); } + append_reasoning(&mut final_reasoning, response_reasoning.as_deref()); // Add websearch tool message if websearch was used if used_websearch { @@ -1693,6 +1825,7 @@ pub async fn run_agent( } else { None }, + reasoning: (!final_reasoning.is_empty()).then_some(final_reasoning), usage: if final_usage.as_ref().map(|u| u.is_empty()).unwrap_or(true) { None } else { @@ -1712,6 +1845,19 @@ fn streaming_requested(streaming: Option) -> bool { streaming.unwrap_or(true) } +/// Add one iteration's thinking to the step's. Every iteration thinks, and a tool-call +/// iteration's thinking is what led to the call, so the result keeps all of them in order, +/// blank-line separated, rather than only the answering turn's. +fn append_reasoning(accumulated: &mut String, reasoning: Option<&str>) { + let Some(reasoning) = reasoning.map(str::trim).filter(|r| !r.is_empty()) else { + return; + }; + if !accumulated.is_empty() { + accumulated.push_str("\n\n"); + } + accumulated.push_str(reasoning); +} + #[cfg(test)] mod tests { use super::*; @@ -1724,6 +1870,16 @@ mod tests { } } + #[test] + fn reasoning_keeps_every_iteration_in_order() { + let mut acc = String::new(); + append_reasoning(&mut acc, Some("I need both cities.\n\n")); + append_reasoning(&mut acc, None); + append_reasoning(&mut acc, Some(" ")); + append_reasoning(&mut acc, Some("Paris is closer.")); + assert_eq!(acc, "I need both cities.\n\nParis is closer."); + } + #[test] fn an_unwritten_streaming_field_streams() { assert!(streaming_requested(None)); @@ -1845,6 +2001,123 @@ mod tests { assert!(matches!(&tools[2].value, ToolValue::Mcp(_))); } + #[test] + fn narrow_roster_keeps_the_entries_a_run_named() { + fn named(id: &str, summary: &str) -> AgentTool { + AgentTool { + id: id.to_string(), + summary: Some(summary.to_string()), + description: None, + value: ToolValue::FlowModule(FlowModuleValue::Script { + input_transforms: HashMap::new(), + path: "u/test/tool".to_string(), + hash: None, + tag_override: None, + is_trigger: None, + pass_flow_input_directly: None, + }), + } + } + fn mcp(id: &str, summary: &str, path: &str) -> AgentTool { + AgentTool { + id: id.to_string(), + summary: Some(summary.to_string()), + description: None, + value: ToolValue::Mcp(windmill_common::flows::McpToolValue { + resource_path: path.to_string(), + include_tools: vec![], + exclude_tools: vec![], + }), + } + } + fn websearch(id: &str, summary: Option<&str>) -> AgentTool { + AgentTool { + id: id.to_string(), + summary: summary.map(str::to_string), + description: None, + value: ToolValue::Websearch(windmill_common::flows::WebsearchToolValue {}), + } + } + let roster = || { + vec![ + named("a", "get_user"), + named("b", "send_email"), + mcp("c", "github", "$res:u/test/gh"), + ] + }; + let names = |tools: &[AgentTool]| -> Vec { + tools.iter().filter_map(|t| t.summary.clone()).collect() + }; + let ids = + |tools: &[AgentTool]| -> Vec { tools.iter().map(|t| t.id.clone()).collect() }; + + // No list at all: the whole roster, as every agent written before the field expects. + assert_eq!( + names(&narrow_roster(roster(), None)), + ["get_user", "send_email", "github"] + ); + + // An empty list is a list: nothing is advertised, and no server is resolved to find that + // out — one that is down must not fail a run that switched it off. + assert!(names(&narrow_roster(roster(), Some(&[]))).is_empty()); + + let enabled = ["get_user".to_string(), "renamed_away".to_string()]; + assert_eq!( + names(&narrow_roster(roster(), Some(&enabled))), + ["get_user"] + ); + // Counted, not quoted: a name is an argument value, and one holding `$var:` arrives as the + // variable's own value, which this log is not masked for. + assert_eq!( + unmatched_enabled_tools_message(&enabled, &["get_user", "send_email", "u/test/gh"]) + .unwrap(), + "--- ENABLED TOOLS: 1 name named no tool of this agent and had no effect ---\n" + ); + + // A server is named by the resource it points at, bare, and never by its summary: that + // label is shown to nobody and two entries may carry the same one. + let named_server = ["u/test/gh".to_string()]; + assert_eq!( + names(&narrow_roster(roster(), Some(&named_server))), + ["github"] + ); + assert!(unmatched_enabled_tools_message(&named_server, &["u/test/gh"]).is_none()); + // Alongside a name that does match, so the summary being rejected is what empties it. + let summary_and_tool = ["get_user".to_string(), "github".to_string()]; + assert_eq!( + names(&narrow_roster(roster(), Some(&summary_and_tool))), + ["get_user"] + ); + assert!(narrow_roster(roster(), Some(&["u/test/other".to_string()])).is_empty()); + + // Web search reaches the model as a provider capability rather than a tool, so it has no + // name of its own and is enabled by a reserved one, whatever label it was authored with. + for label in [None, Some("Web Search")] { + let mut with_websearch = roster(); + with_websearch.push(websearch("w", label)); + assert_eq!( + ids(&narrow_roster( + with_websearch, + Some(&[WEBSEARCH_ENABLED_NAME.to_string()]) + )), + ["w"] + ); + } + } + + #[test] + fn a_tool_cannot_take_the_name_web_search_is_enabled_by() { + // Nothing else in a roster may answer to the reserved name, or enabling that tool would + // switch web search on beside it. Held here rather than by the shape of the name, which is + // an ordinary identifier: the run refuses to start instead. + assert!(TOOL_NAME_REGEX.is_match(WEBSEARCH_ENABLED_NAME)); + assert!(flow_module_tool_name(Some(WEBSEARCH_ENABLED_NAME)).is_err()); + + assert_eq!(flow_module_tool_name(Some("get_user")).unwrap(), "get_user"); + assert!(flow_module_tool_name(Some("get user")).is_err()); + assert!(flow_module_tool_name(None).is_err()); + } + #[test] fn tool_description_prefers_explicit_over_derived_and_name() { assert_eq!( diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index b3324f87ba..b577097c9d 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -408,16 +408,6 @@ pub fn create_empty_dir(path: &PathBuf) -> std::io::Result<()> { } } -/// Signals a detached `spawn_blocking` task that the future awaiting it is -/// gone, so it can stop instead of running to completion in the background. -struct AbortOnDrop(std::sync::Arc); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.store(true, std::sync::atomic::Ordering::Relaxed); - } -} - /// Lay down the tree of an app-backed repository, which git can't clone /// because its URL carries no credential. /// @@ -490,7 +480,7 @@ async fn fetch_repo_archive( // stopping it, so the flag is what a cancelled job uses to reach the // extraction loop. The guard sets it when this future is dropped. let aborted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _abort_on_drop = AbortOnDrop(aborted.clone()); + let _abort_on_drop = crate::common::AbortOnDrop(aborted.clone()); let unpack_archive = download_archive.clone(); tokio::task::spawn_blocking(move || { unpack_repo_archive(&unpack_archive, &download_target, &aborted) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7c112f5985..117fbda086 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; use serde_json::value::RawValue; use uuid::Uuid; -use windmill_parser_ts::remove_pinned_imports; +use windmill_parser_ts::{remove_pinned_import_specifiers, remove_pinned_imports}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo}; @@ -868,7 +868,7 @@ pub async fn install_bun_lockfile( if quiet { Some(&mut quiet_buf) } else { None }, None, ) - .warn_after_seconds(10) + .warn_after_seconds_for(10, "bun install") .await; if quiet && result.is_err() { // On failure, flush suppressed install output so the user can diagnose @@ -1131,9 +1131,12 @@ pub async fn generate_bun_bundle( None, None, ) + .warn_after_seconds_for(60, "bun build") .await?; } else { - let output = Box::into_pin(child_process.wait_with_output()).await?; + let output = Box::into_pin(child_process.wait_with_output()) + .warn_after_seconds_for(60, "bun build") + .await?; if !output.status.success() { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); @@ -1146,6 +1149,81 @@ pub async fn generate_bun_bundle( Ok(()) } +/// [`generate_bun_bundle`], built once more with the version pins dropped from the import +/// specifiers of `main.ts` if it fails. The lockfile pins those versions, but bun fails on a +/// pinned specifier except where it tolerates a failed import (in a `try`, under a `.catch`, in +/// dead code). Such a script builds as written and must keep that bundle, so only failures retry. +async fn generate_bun_bundle_unpinning_imports( + job_dir: &str, + w_id: &str, + job_id: &Uuid, + worker_name: &str, + db: Option<&Connection>, + timeout: Option, + mem_peak: &mut i32, + canceled_by: &mut Option, + common_bun_proc_envs: &HashMap, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, +) -> Result<()> { + let built = generate_bun_bundle( + job_dir, + w_id, + job_id, + worker_name, + db, + timeout, + mem_peak, + canceled_by, + common_bun_proc_envs, + occupancy_metrics, + ) + .await; + // Without a job, a failed build comes back as an `ExecutionErr`; with one, that variant is a + // cancellation or timeout, which must not be retried. + let build_failed = match &built { + Err(error::Error::ExitStatus(..)) => true, + Err(_) => db.is_none(), + Ok(()) => false, + }; + if !build_failed { + return built; + } + let Some(unpinned) = read_file_content(&format!("{job_dir}/main.ts")) + .await + .ok() + .and_then(|main| { + remove_pinned_import_specifiers(&main) + .ok() + .filter(|u| *u != main) + }) + else { + return built; + }; + write_file(job_dir, "main.ts", &unpinned)?; + if let Some(db) = db { + append_logs( + job_id, + w_id, + "\nbundling again with the imports' versions taken from the lockfile\n", + db, + ) + .await; + } + generate_bun_bundle( + job_dir, + w_id, + job_id, + worker_name, + db, + timeout, + mem_peak, + canceled_by, + common_bun_proc_envs, + occupancy_metrics, + ) + .await +} + struct PulledCodebase { is_esm: bool, } @@ -1203,7 +1281,12 @@ async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result = get_common_bun_proc_envs(None).await; - generate_bun_bundle( + generate_bun_bundle_unpinning_imports( job_dir, w_id, job_id, @@ -1321,7 +1404,9 @@ pub async fn prebundle_bun_script( ensure_bundle_output_exists(&origin)?; - save_cache(&local_path, &remote_path, &origin, false).await?; + save_cache(&local_path, &remote_path, &origin, false) + .warn_after_seconds_for(60, "bundle cache save") + .await?; Ok(()) } @@ -1592,7 +1677,9 @@ pub async fn handle_bun_job( } }; - let (cache, logs) = crate::global_cache::load_cache(&local_path, &remote_path, false).await; + let (cache, logs) = crate::global_cache::load_cache(&local_path, &remote_path, false) + .warn_after_seconds_for(60, "bundle cache load") + .await; (cache, logs, local_path, remote_path) } else { (false, "".to_string(), "".to_string(), "".to_string()) @@ -1784,8 +1871,19 @@ pub async fn handle_bun_job( if modules.as_ref().is_some_and(|m| !m.is_empty()) { let bundle_path = std::path::Path::new(job_dir).join("out").join("main.js"); if bundle_path.exists() { + // The lock-generation build kept every `pkg@version` specifier, and bun resolves + // a pinned specifier outside node_modules, loading a second copy of the package. + // The bundle holds the user's code too, so only the specifiers are rewritten, and + // a bundle the parser rejects still runs as built, pins and all. let bundled = std::fs::read_to_string(&bundle_path)?; - write_file(job_dir, "main.ts", &bundled)?; + let unpinned = remove_pinned_import_specifiers(&bundled).unwrap_or_else(|e| { + tracing::warn!( + job_id = %job.id, + "could not unpin the modules bundle, running it as built: {e:#}" + ); + bundled + }); + write_file(job_dir, "main.ts", &unpinned)?; } } "\n\n--- BUN CODE EXECUTION ---\n".to_string() @@ -1899,8 +1997,12 @@ pub async fn handle_bun_job( // Kept comment-free — this string is written out per job. // `_takePendingStepFailure` / `_takePendingSuspend` hand back what the body - // caught and swallowed; honour them instead of reporting a `complete` (see - // `_pendingStepFailure` in client.ts). Optional: npm clients may predate them. + // caught and swallowed; honour them instead of reporting a bare `complete` + // (see client.ts). Optional: npm clients may predate them. + // `_warnUnobservedTaskFailures` reports what the body never awaited, and so + // belongs only on the paths that end the round for good. A round that + // dispatches, sleeps, checkpoints or waits for approval replays later and + // re-registers the same failures from the checkpoint — keep those quiet. let wrapper_content = if is_wac_v2 { format!( r#" @@ -1958,6 +2060,7 @@ async function run() {{ if (trailing.length > 0) {{ return {{ type: "dispatch", mode: trailing.length > 1 ? "parallel" : "sequential", steps: trailing }}; }} + ctx._warnUnobservedTaskFailures?.(); return {{ type: "complete", result: result ?? null }}; }} catch (e) {{ setWorkflowCtx(null); @@ -1970,13 +2073,14 @@ async function run() {{ return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null, started_at: dispatch.started_at, duration_ms: dispatch.duration_ms }}; }} if (dispatch.mode === "approval") {{ - return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled }}; + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled, skin: dispatch.skin, description: dispatch.description }}; }} if (dispatch.mode === "sleep") {{ return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; }} return {{ type: "dispatch", mode: dispatch.mode ?? "sequential", steps: dispatch.steps ?? [] }}; }} + ctx._warnUnobservedTaskFailures?.(); const failed = ctx._takePendingStepFailure?.(); if (failed) {{ throw failed.error; @@ -2185,7 +2289,7 @@ try {{ if !codebase.is_some() && !has_bundle_cache { if build_cache { - generate_bun_bundle( + generate_bun_bundle_unpinning_imports( job_dir, &job.workspace_id, &job.id, @@ -2201,7 +2305,10 @@ try {{ let bundle_path = format!("{job_dir}/main.js"); ensure_bundle_output_exists(&bundle_path)?; if !local_path.is_empty() { - match save_cache(&local_path, &remote_path, &bundle_path, false).await { + match save_cache(&local_path, &remote_path, &bundle_path, false) + .warn_after_seconds_for(60, "bundle cache save") + .await + { Err(e) => { let em = format!("could not save {local_path} to bundle cache: {e:?}"); tracing::error!(em) @@ -2767,15 +2874,17 @@ pub async fn handle_wac_v2_output( .collect(); // Resolve job_payload once (same for all children since they re-run - // the parent script) + // the parent script). The step's cache setting is for the workflow's + // result; a task is cached only through its own `cache_ttl` option, + // under a key of its own (see `cached_result_path`). let job_payload_template = match job.kind { JobKind::Script => { if let Some(hash) = job.runnable_id { Ok(JobPayload::ScriptHash { hash, path: job.runnable_path.clone().unwrap_or_default(), - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), priority: job.priority, @@ -2790,6 +2899,27 @@ pub async fn handle_wac_v2_output( )) } } + // A deployed flow runs an inline step as the `flow_node` its deploy + // rewrote it into; the child re-runs that node the way a `Script` + // child re-runs its hash, so `runnable_id` (the checkpoint's source + // hash) stays the same across parent and children. + JobKind::FlowScript => { + if let Some(id) = job.runnable_id { + Ok(JobPayload::FlowScript { + id: windmill_common::flows::FlowNodeId(id.0), + path: job.runnable_path.clone().unwrap_or_default(), + language: job.script_lang.unwrap_or(ScriptLang::Bun), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: ConcurrencySettings::default(), + }) + } else { + Err(error::Error::internal_err( + "WAC v2 FlowScript job missing runnable_id".to_string(), + )) + } + } JobKind::Preview => { let row: Option<(Option, Option)> = sqlx::query_as( "SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2", @@ -2805,8 +2935,8 @@ pub async fn handle_wac_v2_output( hash: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), lock: lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -2826,6 +2956,26 @@ pub async fn handle_wac_v2_output( { let mut tx = db.begin().await?; + // Park before writing the checkpoint. This locks the queue row ahead of the + // status row, the order `record_child_completion` takes, so a stale child + // finishing while the parent re-dispatches cannot deadlock this transaction. + // A cancel already on the row is also seen before anything is written, and + // every child pushed after the commit finds a parked parent to decrement. + match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + num_steps as i32, + 14.0 * 24.0 * 3600.0, + ) + .await? + { + WacPark::Parked(ms) => segment_ms = ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + } + // Update checkpoint with pending steps update_checkpoint_for_dispatch(&mut checkpoint, &steps, &mode, &job_ids); let status_json = serde_json::to_value(&checkpoint).map_err(|e| { @@ -2875,25 +3025,6 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent before children become visible, so a child that - // completes immediately finds a parked parent to decrement. - match crate::wac_executor::suspend_wac_parent( - &mut tx, - &job.id, - &job.workspace_id, - num_steps as i32, - 14.0 * 24.0 * 3600.0, - ) - .await? - { - WacPark::Parked(ms) => segment_ms = ms, - // Returning here drops `tx`, unwriting the checkpoint and the timeline - // entries, so no child is ever pushed against a parent that never parked. - WacPark::Cancelled(cancel) => { - return Err(wac_cancelled_mid_segment(cancel, canceled_by)) - } - } - tx.commit().await?; } @@ -2904,6 +3035,12 @@ pub async fn handle_wac_v2_output( let mut pushed_ids: Vec = Vec::with_capacity(num_steps); let push_result: error::Result<()> = async { for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + // A task with a runnable of its own (a deployed script or flow) queues + // at that runnable's priority; any other task is the parent's code and + // queues at the parent's. + let own_runnable = matches!(step.dispatch_type.as_str(), "script" | "flow") + && !step.script.starts_with("./"); + // Resolve job payload based on dispatch_type let (job_payload, child_args, is_external, on_behalf_of) = match step.dispatch_type.as_str() { @@ -2917,8 +3054,8 @@ pub async fn handle_wac_v2_output( hash: None, language: module.language, lock: module.lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -3002,7 +3139,8 @@ pub async fn handle_wac_v2_output( let mut job_payload = job_payload; if let Some(cache_ttl) = step.cache_ttl { match &mut job_payload { - JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => { + JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } + | JobPayload::FlowScript { cache_ttl: ref mut ct, .. } => { *ct = Some(cache_ttl) } JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl), @@ -3014,7 +3152,8 @@ pub async fn handle_wac_v2_output( || step.concurrency_time_window_s.is_some() { match &mut job_payload { - JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => { + JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } + | JobPayload::FlowScript { concurrency_settings: ref mut cs, .. } => { if let Some(limit) = step.concurrent_limit { cs.concurrent_limit = Some(limit); } @@ -3080,13 +3219,14 @@ pub async fn handle_wac_v2_output( job.visible_to_owner, step.tag.clone().or_else(|| Some(job.tag.clone())), step.timeout.or(job.timeout), - None, // flow_step_id - step.priority, // priority_override - None, // authed - false, // running - None, // end_user_email - None, // trigger - None, // suspended_mode + None, // flow_step_id + step.priority + .or(if own_runnable { None } else { job.priority }), + None, // authed + false, // running + None, // end_user_email + None, // trigger + None, // suspended_mode ) .await?; @@ -3200,7 +3340,7 @@ pub async fn handle_wac_v2_output( job.id, num_steps ))) } - WacOutput::Approval { key, timeout, form, self_approval_disabled } => { + WacOutput::Approval { key, timeout, form, self_approval_disabled, skin, description } => { let db = match conn { Connection::Sql(db) => db, _ => { @@ -3222,6 +3362,23 @@ pub async fn handle_wac_v2_output( let mut tx = db.begin().await?; + // Park first: the queue row is locked before the status row, the order every + // child completion takes. + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, + timeout_secs, + ) + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; + // Save checkpoint let status_json = serde_json::to_value(&checkpoint).map_err(|e| { error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) @@ -3316,15 +3473,19 @@ pub async fn handle_wac_v2_output( }; // Store approval form metadata for the approval page endpoint - let approval_meta = serde_json::json!({ + let mut approval_meta = serde_json::json!({ "key": key, "form": form, "timeout": timeout_secs as u32, "self_approval_disabled": sad, + "skin": skin.unwrap_or_default(), "resume": resume_url, "cancel": cancel_url, "approvalPage": approval_page_url, }); + if let Some(description) = description.filter(|d| !d.is_null()) { + approval_meta["description"] = description; + } sqlx::query( "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( COALESCE(workflow_as_code_status, '{}'::jsonb), @@ -3372,22 +3533,6 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent with suspend=1 (waiting for 1 approval event) - let segment_ms = match crate::wac_executor::suspend_wac_parent( - &mut tx, - &job.id, - &job.workspace_id, - 1, - timeout_secs, - ) - .await? - { - WacPark::Parked(ms) => ms, - WacPark::Cancelled(cancel) => { - return Err(wac_cancelled_mid_segment(cancel, canceled_by)) - } - }; - tx.commit().await?; crate::wac_executor::end_wac_segment(conn, job, segment_ms); @@ -3425,6 +3570,24 @@ pub async fn handle_wac_v2_output( let mut tx = db.begin().await?; + // Park first: the queue row is locked before the status row, the order every + // child completion takes. suspend=1 (not 0) so the suspended pull query only + // picks it up when `suspend_until <= now()`, not via `suspend <= 0`. + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, + sleep_secs, + ) + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; + // Save checkpoint let status_json = serde_json::to_value(&checkpoint).map_err(|e| { error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) @@ -3473,23 +3636,6 @@ pub async fn handle_wac_v2_output( })?; } - // Use suspend=1 (not 0) so the suspended pull query only picks it up - // when `suspend_until <= now()`, not via `suspend <= 0`. - let segment_ms = match crate::wac_executor::suspend_wac_parent( - &mut tx, - &job.id, - &job.workspace_id, - 1, - sleep_secs, - ) - .await? - { - WacPark::Parked(ms) => ms, - WacPark::Cancelled(cancel) => { - return Err(wac_cancelled_mid_segment(cancel, canceled_by)) - } - }; - tx.commit().await?; crate::wac_executor::end_wac_segment(conn, job, segment_ms); @@ -3527,21 +3673,12 @@ pub async fn handle_wac_v2_output( let source_hash = job.runnable_id.map(|h| h.0.to_string()); let mut tx = db.begin().await?; - crate::wac_executor::persist_inline_checkpoint_delta( - &mut tx, - &job.id, - source_hash.as_deref(), - &key, - value, - started_at.as_deref(), - duration_ms, - ) - .await?; - // Reset running=false so the job is immediately eligible for pickup. // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — // the job should be re-run right away to continue past the cached step. // `prev` holds the pre-update row: RETURNING would see the cleared column. + // Runs before the checkpoint write so the queue row is locked ahead of the + // status row, the order every child completion takes. let segment_ms = sqlx::query_scalar!( "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) UPDATE v2_job_queue q SET running = false, started_at = null @@ -3558,6 +3695,17 @@ pub async fn handle_wac_v2_output( })? .flatten(); + crate::wac_executor::persist_inline_checkpoint_delta( + &mut tx, + &job.id, + source_hash.as_deref(), + &key, + value, + started_at.as_deref(), + duration_ms, + ) + .await?; + tx.commit().await?; crate::wac_executor::end_wac_segment(conn, job, segment_ms); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index f6391eab07..9c012fd117 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -67,6 +67,20 @@ mount { #[cfg(not(debug_assertions))] pub const DEV_CONF_NSJAIL: &str = ""; +/// Tells a `spawn_blocking` task to stop when the future awaiting it goes away. +/// +/// Dropping a `JoinHandle` detaches the task rather than cancelling it, so a +/// cancelled or timed-out phase otherwise leaves the blocking pool working on an +/// answer nobody will read. Hold one of these beside the handle and have the +/// blocking loop check the flag. +pub(crate) struct AbortOnDrop(pub(crate) std::sync::Arc); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.store(true, std::sync::atomic::Ordering::Relaxed); + } +} + /// Turn a JSON value into the string a shell/CLI arg should receive: a JSON string /// becomes its inner value, anything else is re-serialized compactly. pub(crate) fn raw_to_string(x: &str) -> String { @@ -1545,7 +1559,7 @@ pub async fn cached_result_path( client: &AuthedClient, job: &MiniPulledJob, raw_data: Option<&RawData>, -) -> String { +) -> windmill_common::error::Result { let mut hasher = sha2::Sha256::new(); hasher.update(&[job.kind as u8]); if let Some(ScriptHash(hash)) = job.runnable_id { @@ -1560,6 +1574,13 @@ pub async fn cached_result_path( _ => {} } } + // A workflow-as-code task child runs its parent's code with the parent's + // arguments; the step it executes is what tells its result from the parent's + // and from its siblings'. + if let Some(step_key) = wac_executing_key(db, job).await? { + hasher.update(b"wac_step:"); + hasher.update(step_key.as_bytes()); + } hash_args( db, client, @@ -1570,7 +1591,26 @@ pub async fn cached_result_path( job.cache_ignore_s3_path.unwrap_or(false), ) .await; - format!("g/results/{:064x}", hasher.finalize()) + Ok(format!("g/results/{:064x}", hasher.finalize())) +} + +/// The checkpoint step key a workflow-as-code parent seeded for this child at push +/// time; `None` for any job that is not such a child. +async fn wac_executing_key( + db: &DB, + job: &MiniPulledJob, +) -> windmill_common::error::Result> { + if job.parent_job.is_none() || job.flow_step_id.is_some() { + return Ok(None); + } + let key: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint'->>'_executing_key' \ + FROM v2_job_status WHERE id = $1", + ) + .bind(job.id) + .fetch_optional(db) + .await?; + Ok(key.flatten()) } #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/dbt_column_index.rs b/backend/windmill-worker/src/dbt_column_index.rs new file mode 100644 index 0000000000..64f148534b --- /dev/null +++ b/backend/windmill-worker/src/dbt_column_index.rs @@ -0,0 +1,602 @@ +//! Column-level lineage and real column schemas, from the engine's own static +//! analysis. +//! +//! `manifest.json` carries neither. What does is the parquet index an engine +//! writes under `dbt compile --static-analysis strict --write-index`: +//! `dbt.column_lineage.parquet` (column-to-column edges, each labelled `copy`, +//! `mod` or `scan`) and `dbt.node_columns.parquet` (every column of every node, +//! typed and ordered, rather than only the ones an author documented). +//! +//! Four properties shape everything here, all of them measured against the real +//! engines rather than assumed: +//! +//! - **Strict analysis rejects SQL the default accepts.** An unresolvable +//! identifier is an error under `strict` and compiles fine otherwise, so this +//! is a SEPARATE pass with its own `--target-path`, never a flag on the build, +//! and it is opt-in per project. +//! - **A failed pass still writes the index**, holding every edge of the models +//! that did analyze. So the artifact is read whatever the exit status. +//! - **The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 accepts +//! `--write-index`, declares the views over these two tables in its own +//! `views.sql`, and writes neither file; only Fusion does today. Nothing here +//! asks which engine it is beyond "has the flag" — a release that starts +//! writing them is picked up with no change. +//! - **An incremental model has two shapes, and one ingest holds one of them.** +//! `is_incremental()` is false when the target does not exist or the build +//! is `--full-refresh`, so the `{{ this }}` self-join — and any `ref()` inside +//! that branch — compiles only in the other case. What this stores is +//! therefore what the compile in front of it saw: at DEPLOY, before the first +//! build, that is the cold shape, and a project deployed again after its +//! tables exist stores the incremental one for the same source. Nothing here +//! can reconcile that; dbt has no mode that emits both. The flag is taken from +//! the build so a per-run ingest matches its own run, and the version's graph +//! is honest about the compile that produced it rather than about every run +//! that will follow. + +use std::collections::HashSet; +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::path::Path; +use std::time::Duration; + +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::record::{Field, Row}; +use uuid::Uuid; +use windmill_common::dbt_manifest::{ + is_direct, ColumnIndex, IndexedColumn, IngestedColumnEdge, MAX_COLUMN_EDGES, +}; +use windmill_common::error; +use windmill_common::worker::Connection; +use windmill_parser_yaml::dbt::DbtDescriptor; +use windmill_queue::append_logs; + +use crate::dbt_executor::{dbt_command, Invocation, PreparedProject}; +use crate::handle_child::JobCtx; + +/// Where the lineage pass writes, relative to the project directory. +/// +/// Its own tree, not the runtime's `wm_target`: a `dbt compile` writes +/// `manifest.json` and `run_results.json` like any other invocation, and after a +/// build those two are what the graph ingest and `dbt retry` read. +const CLL_ARTIFACTS_DIR: &str = "wm_target_cll"; + +const COLUMN_LINEAGE_PARQUET: &str = "dbt.column_lineage.parquet"; +const NODE_COLUMNS_PARQUET: &str = "dbt.node_columns.parquet"; + +/// Run the lineage pass and read what it produced. +/// +/// Two steps with deliberately different contracts, because conflating them is +/// what made a best-effort annotation able to fail the job it annotates: +/// +/// - [`compile_index`] runs a subprocess and owns the JOB's semantics. Only a +/// cancellation or the job's own deadline can `Err` out of it; a non-zero exit +/// and an over-long output are outcomes, not failures. +/// - [`read_index`] owns the ARTIFACT's semantics. Reading it never fails the +/// job on the artifact's account: an absent, unreadable or partial index is a +/// value, not an error. It runs UNDER the poller all the same, so the job can +/// still end the phase — a cancel, a completion or the phase timeout — which +/// is the job's semantics reaching in, not the artifact's reaching out. +/// +/// The phase budget wraps the compile alone, because it exists to leave the +/// BUILD its share of the clock and only the compile can spend that share +/// unboundedly. The decode's own end is the job's: the poller it runs under +/// stops it when the job stops. +pub(crate) async fn collect( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + // The dbt subcommand the job runs, which decides the effective + // `--full-refresh` — see `dbt_executor::full_refresh`. + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + kept: &HashSet<&str>, +) -> error::Result> { + if !descriptor.column_lineage { + return Ok(None); + } + if !p.engine.engine.writes_column_index() { + append_logs( + job_id, + w_id, + format!( + "\n`column_lineage` is set, but the {} engine has no `--write-index`: column \ + lineage needs an engine that does static analysis. The rest of the graph is \ + unaffected.\n", + p.engine.engine.as_str() + ), + conn, + ) + .await; + return Ok(None); + } + + let index_dir = p.project_dir.join(CLL_ARTIFACTS_DIR).join("index"); + let Some(compiled) = compile_index(p, descriptor, inv, command, ctx, job_id, w_id, conn).await? else { + return Ok(None); + }; + let coverage = Coverage::of(&compiled); + + // Decoded UNDER the poller, not followed by a check of its own. The decode is + // the one phase of this pass with no subprocess behind it, so nothing else + // heartbeats while it runs: left alone, a large index is a silent worker for + // as long as it takes, which the zombie sweep reads as a dead job and + // restarts. The poller pings throughout and ends this with an `Err` if the + // job is cancelled or completed meanwhile — the job's own semantics, which + // this module may always propagate. + let artifact = crate::handle_child::run_future_with_polling_update_job_poller( + *job_id, + ctx.timeout(), + conn, + ctx.mem_peak, + ctx.canceled_by, + async { Ok(read_index(&index_dir, kept).await) }, + ctx.worker_name, + w_id, + &mut Some(ctx.occupancy_metrics), + Box::pin(futures::stream::empty()), + ) + .await?; + + // What only the pass knows. The COUNTS are logged where the index is folded + // into the graph, since the graph decides how much of it is kept. + let note = match artifact { + Artifact::Read(index) => { + if let Some(note) = coverage.caveat() { + log(job_id, w_id, note, &compiled.stderr, conn).await; + } + return Ok(Some(index)); + } + // The truncated arms come first: a compile stopped part-way explains an + // absent or unreadable artifact, and blaming the engine's capability + // for it sends the reader to check the wrong thing entirely. + Artifact::Missing if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass printed more than this runtime reads and was \ + stopped before it wrote `{COLUMN_LINEAGE_PARQUET}`." + ), + Artifact::Unreadable(why) if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass was stopped for printing more than this \ + runtime reads, and the `{COLUMN_LINEAGE_PARQUET}` it had written could not be read \ + ({why})." + ), + // Said apart from the one below, because it sends the reader somewhere + // else entirely: the engine did its job and this runtime could not read + // what it wrote. + Artifact::Unreadable(why) => format!( + "No column lineage: `{COLUMN_LINEAGE_PARQUET}` was written but could not be read \ + ({why}). The graph is unaffected." + ), + Artifact::Missing => format!( + "No column lineage: the analysis pass wrote no `{COLUMN_LINEAGE_PARQUET}`. Only an \ + engine that computes it does, and only for the warehouses it analyzes natively — \ + the flag alone is not the capability." + ), + }; + // The engine's own diagnostics come along. They are how a reader learns that + // this adapter turned static analysis off, which it reports as a warning on + // a SUCCESSFUL compile that nothing else would show. + log(job_id, w_id, ¬e, &compiled.stderr, conn).await; + Ok(None) +} + +async fn log(job_id: &Uuid, w_id: &str, note: &str, stderr: &str, conn: &Connection) { + append_logs( + job_id, + w_id, + format!("\n{note}\n{}", diagnostics(stderr)), + conn, + ) + .await; +} + +/// How completely the analysis compile covered the project. +/// +/// Every way the COMPILE can disappoint is a value here rather than an error. An +/// `Err` from `compile_index` is the JOB's — a cancellation or its deadline — +/// and must fail it; the pass giving up on its own terms is `Ok(None)` and has +/// already been logged. +enum Coverage { + /// Every model analyzed. + Whole, + /// `--static-analysis strict` rejected part of the project. Whatever it did + /// analyze is still in the index. + Partial, + /// The output ceiling killed the compile mid-run. Distinct from `Partial`: + /// nothing rejected the project, but the index is however far it had got, so + /// it is not `Whole` either. + Truncated, +} + +impl Coverage { + fn of(c: &crate::dbt_executor::Captured) -> Self { + match (c.truncated, c.success) { + (true, _) => Coverage::Truncated, + (false, true) => Coverage::Whole, + (false, false) => Coverage::Partial, + } + } + + /// What to tell the reader when an index WAS produced. `None` for a run that + /// covered everything, which needs no caveat. + fn caveat(&self) -> Option<&'static str> { + match self { + Coverage::Whole => None, + Coverage::Partial => Some( + "Column lineage: `--static-analysis strict` rejected part of the project, so \ + the lineage covers only the models it could analyze.", + ), + Coverage::Truncated => Some( + "Column lineage: the analysis pass printed more than this runtime reads and was \ + stopped, so the lineage covers only the models it had reached.", + ), + } + } +} + +/// Run `dbt compile --static-analysis strict --write-index`, under this phase's +/// share of the job's clock. +/// +/// `Ok(None)` is "the pass gave up and said so"; `Err` is the job's own +/// cancellation or deadline and must propagate. Nothing outlives this function: +/// the budget is a race around the child, and dropping that future kills it +/// through `run_captured`'s `kill_on_drop`. +async fn compile_index( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result> { + // A previous pass in the same job directory — a retry's second attempt — + // would otherwise be read back as this one's answer. + tokio::fs::remove_dir_all(p.project_dir.join(CLL_ARTIFACTS_DIR)) + .await + .ok(); + + // Static analysis can log in to read schemas, so it gets live credentials like + // every other dbt process. + if let Err(e) = p.refresh_profile(descriptor, job_id, w_id, conn).await { + append_logs( + job_id, + w_id, + format!("\nColumn lineage: skipped, {e}\n"), + conn, + ) + .await; + return Ok(None); + } + + let mut cmd = dbt_command( + p, + &[ + "compile", + "--static-analysis", + "strict", + "--write-index", + // Documented as what builds the CLL graph, and `--write-index` alone + // happens to imply it on the engine probed. Passed explicitly so the + // pass does not depend on which of the two is doing the work. + "--write-lineage", + "--target-path", + CLL_ARTIFACTS_DIR, + ], + ); + // The flag already wins over the env var dbt_command sets, but setting both + // means this pass cannot write into the runtime's artifacts even if that + // precedence ever changes — and what is in there after a build is the + // `run_results.json` a `dbt retry` resumes from. + cmd.env("DBT_TARGET_PATH", CLL_ARTIFACTS_DIR); + crate::dbt_executor::add_vars(&mut cmd, descriptor, inv)?; + // The BUILD's answer, not the descriptor's default: `is_incremental()` + // branches on it, so a model reading `{{ this }}` compiles its self-join — + // and any `ref()` inside that branch — only when this is absent. Guessing + // here stores lineage for SQL the run never executed. + if crate::dbt_executor::full_refresh(descriptor, inv, command)? { + cmd.arg("--full-refresh"); + } + // Captured rather than streamed: a strict-analysis failure is a wall of + // diagnostics about SQL the build itself accepts, and this pass decides + // nothing about whether that build runs. + // Read before the future below borrows `ctx` mutably. + let budget = phase_budget(ctx); + let run = crate::dbt_executor::run_captured( + cmd, + "dbt compile (column lineage)", + ctx, + job_id, + w_id, + conn, + CLL_MAX_OUTPUT_BYTES, + // The ceiling is this pass's, not the job's: a compile that prints more + // than it than we care to read has still analyzed the project, and the + // index it wrote is on disk either way. + crate::dbt_executor::Overflow::Truncate, + ); + let Some(budget) = budget else { + return Ok(Some(run.await?)); + }; + match tokio::time::timeout(budget, run).await { + Ok(r) => Ok(Some(r?)), + Err(_) => { + append_logs( + job_id, + w_id, + format!( + "\nNo column lineage: the analysis pass did not finish within {}s, half of \ + what was left of this job's time. The build below gets the rest.\n", + budget.as_secs() + ), + conn, + ) + .await; + Ok(None) + } + } +} + +/// stdout the pass may produce. It is a compile, so this is diagnostics rather +/// than data. +const CLL_MAX_OUTPUT_BYTES: usize = 1 << 20; + +/// The share of the job's remaining wall clock this pass may spend. +/// +/// A per-run refresh ingests BEFORE the build and shares the job's one deadline, +/// so an unbounded pass on a slow project would hand `dbt build` an expired +/// budget and fail the run it exists only to annotate. Half leaves the build at +/// least as long as the annotation was allowed to take. +/// +/// Spent as a race around the COMPILE rather than as a shortened deadline handed +/// to the runner: the runner reports its expiry as an `Err`, indistinguishable +/// from a cancellation or the job's own deadline, and those two MUST fail the +/// job. Expiring here is this budget and nothing else. The child dies with the +/// dropped future through `run_captured`'s `kill_on_drop`; the decode is outside +/// this race and answers to the poller instead. +fn phase_budget(ctx: &JobCtx<'_>) -> Option { + ctx.timeout() + .map(|left| Duration::from_secs((left.max(0) as u64 / 2).max(1))) +} + +/// The tail of what the engine said, bounded. The whole of it is every rendered +/// model on a large project, which is not what a job log is for. +const DIAGNOSTIC_LINES: usize = 40; + +fn diagnostics(out: &str) -> String { + let lines: Vec<&str> = out.lines().filter(|l| !l.trim().is_empty()).collect(); + let tail = &lines[lines.len().saturating_sub(DIAGNOSTIC_LINES)..]; + match tail.is_empty() { + true => String::new(), + false => format!("{}\n", tail.join("\n")), + } +} + +/// What came back from the artifact. Never an `Err`: nothing the file does or +/// fails to do is a reason to fail a job. `Unreadable` is separate from +/// `Missing` because the two send a reader looking in different places — one at +/// their engine and adapter, the other at a file that exists. +enum Artifact { + Read(ColumnIndex), + Missing, + Unreadable(String), +} + +/// Read both parquets, if the lineage one is there. +/// +/// The column schemas alone are not worth a graph: they arrive with the lineage +/// or not at all, and a node's declared columns already answer for the case +/// where the pass never ran. +async fn read_index(index_dir: &Path, kept: &HashSet<&str>) -> Artifact { + let lineage = index_dir.join(COLUMN_LINEAGE_PARQUET); + if !tokio::fs::try_exists(&lineage).await.unwrap_or(false) { + return Artifact::Missing; + } + let columns = index_dir.join(NODE_COLUMNS_PARQUET); + // Owned, because the decode moves to a blocking thread. The index describes + // the whole project while this graph describes one selection of it, so + // scoping HERE is what keeps the bound below from being spent on rows the + // graph would discard anyway. + let kept: HashSet = kept.iter().map(|s| (*s).to_string()).collect(); + // Dropping the handle of a blocking task does NOT stop it: the poller + // cancelling this phase would otherwise leave a thread decoding millions of + // rows for a job that is over. `abandoned` is set when this future is + // dropped, and the row loop reads it. + let abandoned = Arc::new(AtomicBool::new(false)); + let _stop = crate::common::AbortOnDrop(abandoned.clone()); + // Decompressing and decoding a parquet is CPU work on a file the engine just + // wrote, so it does not belong on the runtime's poll thread. + let read = tokio::task::spawn_blocking(move || { + read_index_blocking(&lineage, &columns, &kept, &abandoned) + }) + .await; + match read { + Ok(Ok(index)) => Artifact::Read(index), + Ok(Err(e)) => Artifact::Unreadable(e.to_string()), + Err(e) => Artifact::Unreadable(e.to_string()), + } +} + +fn read_index_blocking( + lineage: &Path, + columns: &Path, + kept: &HashSet, + abandoned: &AtomicBool, +) -> error::Result { + let mut out = ColumnIndex::default(); + // ONE pass, with the two kinds bucketed as they arrive. `copy` and `mod` say + // the value itself travelled, so they get the whole budget; `scan` — the + // column was read to produce the ROW, which reaches every output column of + // its model and is the bulk of a wide project's index — fills only what is + // left over at the end. Reading the file twice to get that ordering would + // double the decode of exactly the large index this bound exists for. + let mut scan: Vec = Vec::new(); + for_each_row(lineage, abandoned, |row| { + let lineage_kind = string(row, "lineage_kind"); + let parent_unique_id = string(row, "from_node_unique_id"); + let child_unique_id = string(row, "to_node_unique_id"); + let parent_column = string(row, "from_column_name"); + let child_column = string(row, "to_column_name"); + // A column of a node the analysis could not name is not an endpoint the + // graph can draw, and neither is one outside this graph's nodes. + if parent_column.is_empty() + || child_column.is_empty() + || !kept.contains(&parent_unique_id) + || !kept.contains(&child_unique_id) + { + return ControlFlow::Continue(()); + } + let edge = IngestedColumnEdge { + parent_unique_id, + parent_column, + child_unique_id, + child_column, + lineage_kind, + }; + // The bound covers BOTH buckets, so the pass never holds more than one + // budget's worth however the kinds are distributed. + let held = out.edges.len() + scan.len(); + if is_direct(&edge.lineage_kind) { + // A direct edge displaces a `scan` one: the budget is spent on + // value flow first. + if held >= MAX_COLUMN_EDGES { + scan.pop(); + } + out.edges.push(edge); + // The edge that FILLS the budget ends the read, not the next one to + // arrive: once the displacing kind is full nothing later in the file + // can be kept, and waiting for another direct edge to say so decodes + // a `scan`-only tail all the way to the backstop for nothing. + return match out.edges.len() >= MAX_COLUMN_EDGES { + true => ControlFlow::Break(()), + false => ControlFlow::Continue(()), + }; + } + if held < MAX_COLUMN_EDGES { + scan.push(edge); + } + // Not a stopping point even when full: a direct edge still to come takes + // a `scan` entry's place. + ControlFlow::Continue(()) + })?; + out.edges.append(&mut scan); + // Absent is normal — an engine can write the lineage table and not this one — + // and unreadable is not worth losing the lineage over. + let mut held = 0usize; + let _ = for_each_row(columns, abandoned, |row| { + let unique_id = string(row, "unique_id"); + let name = string(row, "column_name"); + if held >= MAX_INDEXED_COLUMNS { + return ControlFlow::Break(()); + } + if name.is_empty() || !kept.contains(&unique_id) { + return ControlFlow::Continue(()); + } + held += 1; + // The author's `data_type` where `schema.yml` gives one, since that is + // what the project calls the column; the analysis's own inference + // otherwise. + let column_type = match string(row, "declared_type") { + t if !t.is_empty() => t, + _ => string(row, "inferred_type"), + }; + out.columns + .entry(unique_id) + .or_default() + .push(IndexedColumn { + name, + column_type, + index: int(row, "column_index").unwrap_or(i64::MAX), + }); + ControlFlow::Continue(()) + }); + Ok(out) +} + +/// The most rows of `dbt.node_columns.parquet` one pass keeps. One per column of +/// the project, so the same bound as the edges is far more than any project +/// reaches; it exists for the same reason. +const MAX_INDEXED_COLUMNS: usize = MAX_COLUMN_EDGES; + +/// The most rows of an index one pass DECODES, whatever it keeps of them. +/// +/// A bound on work rather than on memory, and the two are separate because the +/// input this defends against is the one that cannot be collected: `scan` +/// lineage is emitted from every predicate and join column to every output +/// column, so a project shaped that way writes an index whose row count is +/// quadratic in its widest model. This pass runs outside the phase budget, on a +/// blocking thread, and nothing the file contains may fail a deploy or a run — +/// so the file it walks needs an end even when almost nothing in it is +/// retained. The abandonment flag ends it sooner when the job is over; this is +/// the bound for a job that is not. +const MAX_INDEX_ROWS: usize = 4_000_000; + +/// Decode a parquet a row at a time, handing each to `f` and never holding two. +/// +/// Collecting first would put a `Vec` — each row carrying its own copy of +/// every column NAME — in front of the caller's own bound, which is what would +/// take the worker process down on the index described above. +/// +/// `f` says when it has all it will take, and that is the ordinary end: this +/// runs outside the phase budget, so every row decoded past the point of being +/// able to keep one is wall clock the build below does not get. +fn for_each_row( + path: &Path, + abandoned: &AtomicBool, + mut f: impl FnMut(&Row) -> ControlFlow<()>, +) -> error::Result<()> { + let fail = |e: parquet::errors::ParquetError| { + error::Error::internal_err(format!("reading {}: {e}", path.display())) + }; + let file = std::fs::File::open(path) + .map_err(|e| error::Error::internal_err(format!("opening {}: {e}", path.display())))?; + let reader = SerializedFileReader::new(file).map_err(fail)?; + for (n, row) in reader.get_row_iter(None).map_err(fail)?.enumerate() { + // Nobody is waiting for this any more — the job was cancelled, completed + // or ran out of time while it decoded. + if abandoned.load(Ordering::Relaxed) { + break; + } + if n >= MAX_INDEX_ROWS { + tracing::warn!( + "dbt column index: {} holds more than {MAX_INDEX_ROWS} rows; the rest is dropped", + path.display() + ); + break; + } + if f(&row.map_err(fail)?).is_break() { + break; + } + } + Ok(()) +} + +/// By NAME, not by position: these tables are the engine's own schema and it +/// adds columns to them between releases. +fn field<'a>(row: &'a Row, name: &str) -> Option<&'a Field> { + row.get_column_iter() + .find(|(k, _)| k.as_str() == name) + .map(|(_, v)| v) +} + +fn string(row: &Row, name: &str) -> String { + match field(row, name) { + Some(Field::Str(s)) => s.clone(), + Some(Field::Bytes(b)) => String::from_utf8_lossy(b.data()).into_owned(), + _ => String::new(), + } +} + +fn int(row: &Row, name: &str) -> Option { + match field(row, name) { + Some(Field::Long(v)) => Some(*v), + Some(Field::Int(v)) => Some(*v as i64), + Some(Field::Short(v)) => Some(*v as i64), + Some(Field::UInt(v)) => Some(*v as i64), + Some(Field::ULong(v)) => i64::try_from(*v).ok(), + _ => None, + } +} diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 33ddd6c32e..8119897d6d 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -19,12 +19,13 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::client::AuthedClient; use windmill_common::error::{self, Error}; +use windmill_common::jobs::JobKind; use windmill_common::materialization::{ record_materialization, MaterializationStatus, RecordMaterializationRequest, }; use windmill_common::worker::{to_raw_value, write_file, Connection}; use windmill_parser_yaml::{ - parse_dbt_descriptor, DbtDescriptor, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, + parse_dbt_descriptor, DbtDescriptor, DbtEngine, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, DBT_COMMAND_LABEL, DBT_DEFAULT_WAREHOUSE, }; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -37,6 +38,9 @@ use crate::dbt_engine::{provision_engine, ProvisionedEngine, DBT_CACHE_DIR}; use crate::dbt_profiles::{ ensure_adapter_licensed, render_dbt_profile, render_profile, DbtAdapter, KnownAdapter, }; +use crate::dbt_state::{ + environment_label, prepare_deferral, write_state_dir, Deferral, StateManifest, STATE_DIR, +}; use crate::handle_child::{ get_mem_peak, handle_child, run_future_with_polling_update_job_poller, JobCtx, JobDeadline, }; @@ -121,6 +125,12 @@ pub struct DbtRunResult { /// the same project — cannot get them from the job. #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub invocation_args: std::collections::HashMap>, + /// The run whose stored state this one resolved its unbuilt `ref()`s + /// through, absent when it deferred to none. What a deferring run built + /// against is otherwise unrecoverable: the state is replaced by the next + /// successful run of that environment. + #[serde(skip_serializing_if = "Option::is_none")] + pub deferred_to: Option, } #[derive(Serialize, Debug, Default)] @@ -189,7 +199,13 @@ pub(crate) async fn handle_dbt_job( // result publishes, and both describe an invocation of this script, not one // executor's view of it. let raw_args = job.args.as_ref().map(|a| a.0.clone()).unwrap_or_default(); - let inv = Invocation { args: args.clone(), raw_args, envs: envs.clone(), strict: true }; + let inv = Invocation { + args: args.clone(), + raw_args, + envs: envs.clone(), + deferral: None, + strict: true, + }; // One wall clock for the whole job. A dbt job is a sequence of // subprocesses — provision, deps, parse, ls, build, then the // `after_all` tests — and each would otherwise resolve the job's full @@ -262,6 +278,16 @@ pub(crate) async fn handle_dbt_job( // applies — nothing is built, so there is no test phase, no materialization, // no retry state and no ownership to publish. if command == "parse" { + // Checked here rather than at the seam below, which cannot tell a parse + // from a run that simply left `defer` off: a parse never reaches the + // deferral at all, so it is the one caller for which "turn `defer` on" + // would be advice that leads nowhere. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + StateAccess::Never(&command), + !selection_is_overridden(&descriptor, &inv.args)?, + )?; return run_parse_only( &prepared, &descriptor, @@ -364,6 +390,84 @@ pub(crate) async fn handle_dbt_job( inv }; + // Read AFTER the retry restore, so a retry defers exactly as the run it + // resumes did: a retry's own arguments are the command block alone, and the + // relations its unbuilt `ref()`s resolve to must not depend on that. + let defer = arg_bool(&inv.args, "defer")?.unwrap_or(descriptor.defer); + // Before the state is fetched, not only at the seam where the selection + // reaches dbt: a selector that cannot work whatever the state says would + // otherwise be masked by the "nothing published yet" refusal, which sends the + // caller to publish a state that will not help. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + if defer { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(&descriptor, &inv.args)?, + )?; + // A `show` defers too, and every engine takes the flags on it: it COMPILES + // the model it previews, so a model whose upstream this environment built and + // this run did not is exactly the case a deferral exists for. + let inv = if defer { + // Refused before anything runs. `dbt retry` reads the run it resumes + // from `--state`, the flag a deferral needs, so an engine without + // `--defer-state` can be given one or the other: told to defer, it + // resumes the stored state's own (successful) results and rebuilds + // nothing, and left alone it rebuilds the failed nodes with every + // `ref()` resolving into the schema THIS run writes — which for the + // narrowed run a deferral exists to serve is not where those models go. + if command == "retry" && !prepared.engine.engine.has_defer_state_flag() { + return Err(Error::BadRequest(format!( + "`{}` cannot resume a run that deferred: `dbt retry` takes the run it resumes \ + from `--state`, which is also where a deferral reads its manifest, and this \ + engine has no `--defer-state` to tell the two apart. Run the script again \ + instead of resuming it, or move the project to dbt-core-1x", + prepared.engine.engine.as_str() + ))); + } + let deferral = prepare_deferral(&prepared, &job.workspace_id, job_dir, conn).await?; + // Only answerable once the state is loaded: `defer` is enough for a + // `state:` method, which reads the manifest every publication carries, + // but a `result:` one reads `run_results.json` — and a build recovered by + // node retry publishes without it, since the results it holds describe + // only the nodes the retry rebuilt. dbt-core then raises an INTERNAL + // error and the Rust engines match nothing and exit 0. + if !deferral.has_run_results + && selection_names( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + &["result"], + ) + { + return Err(Error::BadRequest(format!( + "a `result:` selector reads `run_results.json` out of the published state, and \ + the state for this environment ({}) carries only the manifest run {} \ + published: a build recovered by node retry stores none, its results describing \ + the retried nodes rather than the whole build. Run this script once without \ + `defer` and without overrides to publish a complete state, or drop the selector", + environment_label(&prepared), + deferral.published_by + ))); + } + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nDeferring unbuilt refs to the dbt state published by run {}; this run \ + publishes none of its own\n", + deferral.published_by + ), + conn, + ) + .await; + Invocation { deferral: Some(deferral), ..inv } + } else { + inv + }; + // Ingested BEFORE the build, from a `dbt parse` with this run's vars, so the // models shown are the ones about to be built. Rows are keyed by path, version // AND job so no two runs collide; the path-keyed `asset` usage belongs to one @@ -387,7 +491,7 @@ pub(crate) async fn handle_dbt_job( // For a retry the restored manifest already describes the invocation // being resumed, so only the ingest runs — with that invocation's // arguments, which the selection resolver needs to interpolate. - ingest_from_run(&prepared, &descriptor, &inv, &mut ctx, job, conn).await?; + ingest_from_run(&prepared, &descriptor, &inv, &command, &mut ctx, job, conn).await?; } // A read-only command prints rows to stdout, so it is captured rather than @@ -428,9 +532,28 @@ pub(crate) async fn handle_dbt_job( // previous attempt's `run_results.json` is still in the job directory. Never on // an agent worker, which cannot read `v2_job_queue` — the wait below would be // uninterruptible, so a cancelled job would hold its slot and then start dbt. + // And never where the engine cannot be told to defer on a `retry`: the + // rebuild would resolve this run's unbuilt refs into the schema it writes + // into, so the nodes it "recovered" would read from the wrong relations. + // Said out loud below rather than silently skipped. + let retry_would_lose_the_deferral = + inv.deferral.is_some() && !prepared.engine.engine.has_defer_state_flag(); let node_retry = descriptor .retry_failed_nodes - .filter(|_| matches!(conn, Connection::Sql(_))); + .filter(|_| matches!(conn, Connection::Sql(_))) + .filter(|_| !retry_would_lose_the_deferral); + if descriptor.retry_failed_nodes.is_some() && retry_would_lose_the_deferral { + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nSkipping the automatic node retry: `{}` cannot defer on a `dbt retry`\n", + prepared.engine.engine.as_str() + ), + conn, + ) + .await; + } let mut retries_left = node_retry.map(|p| p.attempts()).unwrap_or(0); if let Some(policy) = node_retry.filter(|_| run.is_err()) { retry_failed_nodes( @@ -519,6 +642,56 @@ pub(crate) async fn handle_dbt_job( { tracing::warn!("dbt: could not save retry state for job {}: {e:#}", job.id); } + // What a later run defers to, published by the runs whose relations are the + // SCRIPT's — the same condition that decides whether a run's graph becomes + // what the script owns, and for the same reason: an invocation that scoped + // its own models has no standing to say where this project's relations live. + // Success is the other half, because a relation a deferral resolves to has + // to exist. A `retry` is excluded: its `run_results.json` names only the + // nodes it redid, so publishing it would leave the environment claiming a + // run of a handful of models. + // + // And never a run that DEFERRED, whatever narrowed it. A deferring run built + // some of the relations its manifest names and resolved the rest out of the + // state it read, so publishing that manifest would record relations nothing + // built — and a model renamed since would be recorded under a name only a + // full build creates, breaking every later deferral until one repairs it. + // `publishes_ownership` cannot see this on its own: it reads the caller's + // overrides, and a descriptor that already narrows `select` needs none. + if run.is_ok() + && command == "build" + && inv.deferral.is_none() + // A run of the DEPLOYED version, by kind. A preview carries a + // caller-supplied `script_hash` into `runnable_id` + // (`run_preview_script`), so the version guard alone would let anyone who + // may run a job publish arbitrary content as a deployed script's state. + && job.kind == JobKind::Script + && prepared.graph_refresh.publishes_ownership() + { + // Losing it costs the next deferral, not the run that just finished — + // but silently, so the one actionable case (an artifact too large for + // the database on an instance with no object storage) says so. + if let Err(e) = crate::dbt_state::publish( + &prepared, + &job.workspace_id, + &job.id, + job.runnable_id.map(|h| h.0), + // An attempt was spent, so `run_results.json` on disk is the one + // `dbt retry` left: the nodes it redid, not the build. + node_retry.is_some_and(|p| retries_left < p.attempts()), + conn, + ) + .await + { + append_logs( + &job.id, + &job.workspace_id, + format!("\nCould not publish this run as the environment's dbt state: {e}\n"), + conn, + ) + .await; + } + } let reconciled = reconcile_materializations(&prepared, &results, job, conn, client).await; terminalize_running_relations(job, &reconciled, conn).await; @@ -653,12 +826,26 @@ pub(crate) async fn dbt_dep( None => GraphPublisher::Unversioned, }; let superseded = if let Some(warehouse) = prepared.warehouse.as_deref() { - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, prepared.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + &prepared, + &descriptor, + &inv, + // A deploy resolves the project by parsing it; nothing is built, so + // the pass takes the descriptor's own answer. + "parse", + &mut ctx, + job_id, + w_id, + &conn, + ) + .await?; let published = persist_ingest( db, w_id, @@ -682,6 +869,7 @@ pub(crate) async fn dbt_dep( &conn, ) .await; + warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await; } !published } else { @@ -866,6 +1054,9 @@ impl GraphRefresh { if selection_is_overridden(descriptor, args)? { self.per_run_models = true; } + if full_refresh_is_overridden(descriptor, args)? { + self.per_run_models = true; + } Ok(()) } } @@ -894,6 +1085,17 @@ pub struct PreparedProject { /// The descriptor's `profile.target`, passed as `--target` so it applies to /// a project-owned `profiles.yml` as well as a rendered one. pub target: Option, + /// The target dbt actually runs, which is the above only when the descriptor + /// names one: otherwise it is the workspace warehouse's, or the project's own + /// `profiles.yml` default. Half of an environment's identity, since a + /// `target.name` macro decides where a model is built. + pub effective_target: Option, + /// Whether the profile templates where its relations go — a project-owned + /// `profiles.yml`, a `dbt_profile` resource's block, or `profile.schema`, + /// all of which reach dbt as written. Two renderings then share one + /// `relation_root` and an environment cannot be told apart, so such a + /// project neither publishes state nor defers to any. + pub templated_location: bool, /// The profile target's database. Nodes that override it qualify their /// `dbt://` schema segment so two databases cannot collapse onto one node. pub default_database: Option, @@ -919,19 +1121,76 @@ pub struct PreparedProject { /// Written nsjail profile for this job, when the worker sandboxes jobs. /// `None` means the phases run unsandboxed, exactly as before. pub sandbox_config: Option, - /// One-way digest of the rendered profile — the resolved connection, not - /// just the names it exposes. A resource repointed from one warehouse to - /// another that happens to use the same database and schema names is - /// invisible to `relation_root`, and a retry would then execute the saved - /// failures against a warehouse where the successful nodes do not exist. + /// One-way digest of the rendered profile, credentials masked: the resolved + /// connection, not just the names it exposes. A resource repointed from one + /// warehouse to another that happens to use the same database and schema + /// names is invisible to `relation_root`, and a retry would then execute the + /// saved failures against a warehouse where the successful nodes do not exist. pub profile_digest: String, + /// The job's client, which `refresh_profile` re-resolves the warehouse with. + client: AuthedClient, } impl PreparedProject { + /// Re-resolve the warehouse and rewrite `profiles.yml` just before a dbt + /// process that logs in. What preparation wrote can have expired by then: a + /// Snowflake OAuth token lasts ten minutes, and the build follows `dbt deps` + /// and a parse, the `after_all` tests follow the build, a node retry follows + /// its backoff. + pub(crate) async fn refresh_profile( + &self, + descriptor: &DbtDescriptor, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + ) -> error::Result<()> { + // A project-owned `profiles.yml` is rendered by dbt itself, from an + // environment resolved once. + if descriptor.profile.profiles_yml.is_some() { + return Ok(()); + } + let fresh = match write_profiles( + descriptor, + &self.project_dir, + &self.project_dir.to_string_lossy(), + &self.client, + &self.template_env(), + ) + .await + { + Ok(fresh) => fresh, + // The profile on disk is still whole: a credential that does not + // expire connects with it exactly as before. + Err(e) => { + append_logs( + job_id, + w_id, + format!( + "\nCould not re-resolve the warehouse, so this dbt process uses the \ + credentials resolved earlier in the job: {e}\n" + ), + conn, + ) + .await; + return Ok(()); + } + }; + // Credentials are masked out of the digest, so a mismatch is the warehouse + // itself moving mid-run, and this process would build somewhere else. + if fresh.digest != self.profile_digest { + return Err(Error::BadRequest(format!( + "the `{}` warehouse was repointed while this run was in progress; run the \ + script again", + self.warehouse.as_deref().unwrap_or(DBT_DEFAULT_WAREHOUSE) + ))); + } + Ok(()) + } + /// Where this run's relations live: the resolved schema and database. Drift /// here since the deploy means the stored graph names relations that no /// longer exist. - fn relation_root(&self) -> String { + pub(crate) fn relation_root(&self) -> String { format!( "{}|{}", self.default_schema.as_deref().unwrap_or(""), @@ -1063,8 +1322,8 @@ pub(crate) async fn prepare_project( .chain(invocation_env.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect(); - let (profiles_dir, warehouse, adapter, default_database, default_schema, profile_digest) = - write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let profile = write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let adapter = profile.adapter.clone(); // The lockfile's version, when it pinned one for this same engine — a // descriptor edited to another engine invalidates the pin. let pinned_version = locks @@ -1185,18 +1444,21 @@ pub(crate) async fn prepare_project( h.finish() }, sandbox_config, - profile_digest, + profile_digest: profile.digest, + client: client.clone(), project_dir, - profiles_dir, + profiles_dir: profile.dir, engine, graph_refresh, - warehouse, + warehouse: profile.warehouse, target: descriptor.profile.target.clone(), + effective_target: profile.target, + templated_location: profile.templated_location, descriptor_content: descriptor_content.to_string(), descriptor_env, - default_database, - default_schema, + default_database: profile.database, + default_schema: profile.schema, script_path: script_path.to_string(), env, }; @@ -1511,6 +1773,24 @@ async fn strip_git_remote(dir: &Path) -> std::io::Result<()> { tokio::fs::write(&config, out).await } +/// What resolving the run's connection settled, beyond the file itself. +struct ResolvedProfile { + dir: PathBuf, + /// The workspace warehouse's NAME, when this project belongs to one. + warehouse: Option, + adapter: DbtAdapter, + database: Option, + schema: Option, + /// The target dbt actually runs, which is not always the descriptor's: it + /// falls back to the workspace warehouse's, and to the project's own + /// `profiles.yml` default. Resolved because it is half of an environment's + /// identity and a `target.name` macro can move every relation. + target: Option, + /// Whether a project-owned `profiles.yml` templates where its relations go. + templated_location: bool, + digest: String, +} + /// Write `profiles.yml`, either rendered from a Windmill resource or taken from /// the project itself. Both paths are supported (decision 8): the workspace /// warehouse is the ergonomic one, the project's own file is what makes an @@ -1521,14 +1801,7 @@ async fn write_profiles( job_dir: &str, client: &AuthedClient, template_env: &HashMap, -) -> error::Result<( - PathBuf, - Option, - DbtAdapter, - Option, - Option, - String, -)> { +) -> error::Result { // The workspace's warehouse, always: a descriptor names one by NAME or takes // `main`, and cannot name a resource at all. The NAME is what asset identity // keys on, so every project on one warehouse shares its nodes while the @@ -1609,14 +1882,16 @@ async fn write_profiles( } None => None, }; - return Ok(( + return Ok(ResolvedProfile { dir, - identity, + warehouse: identity, adapter, - target.database, - target.schema, - profile_digest, - )); + database: target.database, + schema: target.schema, + target: Some(target.name), + templated_location: target.templated_location, + digest: profile_digest, + }); } use windmill_common::workspaces::DBT_PROFILE_RESOURCE_TYPE; @@ -1666,9 +1941,6 @@ async fn write_profiles( .or(workspace_target.as_deref()) .unwrap_or("default"); let dir = PathBuf::from(job_dir).join("dbt_profiles"); - tokio::fs::create_dir_all(&dir) - .await - .map_err(|e| Error::internal_err(format!("creating the profiles dir: {e}")))?; let rendered = if is_dbt_profile { let block = value.as_object().ok_or_else(|| { Error::BadRequest( @@ -1689,6 +1961,7 @@ async fn write_profiles( } else { render_profile( &adapter, + descriptor.engine(), &value, &profile_name, target, @@ -1697,6 +1970,10 @@ async fn write_profiles( &dir, )? }; + // After the render, so a render that fails leaves the previous profile whole. + fresh_dir(&dir) + .await + .map_err(|e| Error::internal_err(format!("creating the profiles dir: {e}")))?; write_file(dir.to_str().unwrap(), "profiles.yml", &rendered.yaml)?; if let Some(pem) = rendered.root_certificate_pem.as_deref() { write_file( @@ -1706,19 +1983,41 @@ async fn write_profiles( )?; } let profile_digest = profile_identity_digest( - &rendered.yaml, + &rendered.identity, &dir, rendered.root_certificate_pem.as_deref(), &client.token, ); - Ok(( + Ok(ResolvedProfile { dir, - Some(warehouse.to_string()), + warehouse: Some(warehouse.to_string()), adapter, - rendered.database, - rendered.schema, - profile_digest, - )) + // A `dbt_profile` resource is one block of the user's own + // `profiles.yml`, copied through unchanged, and `profile.schema` is + // written as given — so either can be a template dbt renders and this + // runtime does not, exactly as a project-owned file can. + templated_location: [rendered.database.as_deref(), rendered.schema.as_deref()] + .iter() + .any(|v| v.is_some_and(is_jinja)), + database: rendered.database, + schema: rendered.schema, + target: Some(target.to_string()), + digest: profile_digest, + }) +} + +/// An empty directory at `dir`, whatever was there. `refresh_profile` writes into +/// the job directory after project code has run in a jail that can write it, and a +/// symlink left at `dir` or inside it would carry the worker's write out of the +/// sandbox. An entry that is not a real directory is unlinked, never followed. +async fn fresh_dir(dir: &Path) -> std::io::Result<()> { + match tokio::fs::symlink_metadata(dir).await { + Ok(m) if m.is_dir() => tokio::fs::remove_dir_all(dir).await?, + Ok(_) => tokio::fs::remove_file(dir).await?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + tokio::fs::create_dir_all(dir).await } /// Where a workspace warehouse name points: its resource path and, if the @@ -1740,7 +2039,9 @@ async fn resolve_warehouse( .map_err(|e| Error::BadRequest(format!("resolving the dbt warehouse `{warehouse}`: {e}"))) } -/// Identifies the connection a rendered profile describes, for run identity. +/// Identifies the connection a rendered profile describes, for run identity, +/// from the rendering whose credentials are already masked +/// (`RenderedProfile::identity`). /// /// Two things in the rendered text belong to the ATTEMPT rather than the /// connection, and hashing either as-is makes a retry reject its own @@ -1859,13 +2160,45 @@ async fn adapter_from_profiles_yml( // identically to one on a workspace warehouse, which is what lets the two // meet on the same node when they are on the same relation. let (database_key, schema_key) = adapter.target_identity_keys(); - let read = |k: &str| { + let raw = |k: &str| { out.get(k) .and_then(|v| v.as_str()) - .map(|v| v.to_string()) - .filter(|v| !v.is_empty() && !v.contains("{{")) + .filter(|v| !v.is_empty()) }; - Ok(ProfileTarget { adapter, database: read(database_key), schema: read(schema_key) }) + let read = |k: &str| raw(k).filter(|v| !v.contains("{{")).map(|v| v.to_string()); + Ok(ProfileTarget { + adapter, + database: read(database_key), + schema: read(schema_key), + // A TEMPLATED location is one dbt renders and this runtime does not, so + // two renderings of this file resolve to one `relation_root` and would + // share one environment — `{{ }}` because `read` drops it and it reads + // as absent, `{% %}` because the raw block is kept and reads the same + // for every rendering. Distinguished from plainly absent, which is the + // adapter's default and does not move. + templated_location: [database_key, schema_key] + .iter() + .any(|k| raw(k).is_some_and(is_jinja)), + // The output actually chosen, which for a templated `target:` is the sole + // one rather than the template text no output answers to. + name: match ( + templated_target, + outputs.as_mapping().and_then(|m| m.keys().next()), + ) { + (true, Some(only)) => only.as_str().unwrap_or(target).to_string(), + _ => target.to_string(), + }, + }) +} + +/// Whether dbt would RENDER this value rather than take it literally. +/// +/// Both delimiters, because dbt renders a profile through Jinja: `{{ … }}` +/// substitutes and `{% … %}` branches, and a schema spelled +/// `{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}` moves +/// every relation exactly as an `env_var()` does. +fn is_jinja(v: &str) -> bool { + v.contains("{{") || v.contains("{%") } /// What a project-owned `profiles.yml` target says, for the two things Windmill @@ -1876,6 +2209,14 @@ struct ProfileTarget { adapter: DbtAdapter, database: Option, schema: Option, + /// The output this resolved to, by name. + name: String, + /// Whether its database or schema is a template rather than a literal. The + /// fields above cannot say: a `{{ }}` value is dropped and reads as absent, + /// a `{% %}` block is kept and reads the same for every rendering. So this + /// is what separates "the adapter's default, which does not move" from + /// "wherever this run's environment renders it to". + templated_location: bool, } lazy_static::lazy_static! { @@ -2006,7 +2347,11 @@ pub(crate) fn dbt_command(p: &PreparedProject, args: &[&str]) -> Command { .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("GIT_PATH", GIT_PATH.as_str()); + .env("GIT_PATH", GIT_PATH.as_str()) + // dbt reports anonymous usage to dbt Labs from every invocation unless told + // not to, and a project's `flags:` block cannot override the variable. Set + // before the project's environment so a descriptor can still opt back in. + .env("DBT_SEND_ANONYMOUS_USAGE_STATS", "false"); // Both environments belong to the child. Under a sandbox they reach it through // the jail profile instead: set here, they would reach the dynamic loader that // execs nsjail itself, so an `LD_PRELOAD` from the project would run as the @@ -2191,6 +2536,29 @@ async fn retry_failed_nodes( } } +/// The flags that point a deferring invocation at its state directory. +/// +/// `--state` is where a deferred `ref()` resolves through — except on a `retry`, +/// which reads the run it RESUMES from that same flag: handed the deferral's +/// directory, dbt resumes the successful run stored there and rebuilds nothing. +/// dbt-core 1.x has `--defer-state` for exactly this split; the Rust engines do +/// not, and a run that defers is refused a retry there rather than rebuilt with +/// its refs resolving into the schema it writes into (`handle_dbt_job`), which +/// is why the last arm never fires in practice. +/// +/// The directory is relative because dbt records the invocation's flags into +/// `run_results.json`: an absolute path would name the job directory of the run +/// being resumed, gone by the time anything reads it back. +fn defer_flags(command: &str, engine: DbtEngine) -> &'static [&'static str] { + match command { + // `--defer` itself is restored with the rest of the resumed + // invocation's arguments and cannot be set from here. + "retry" if engine.has_defer_state_flag() => &["--defer-state", STATE_DIR], + "retry" => &[], + _ => &["--defer", "--state", STATE_DIR], + } +} + #[allow(clippy::too_many_arguments)] async fn run_dbt( p: &PreparedProject, @@ -2202,6 +2570,8 @@ async fn run_dbt( ctx: &mut JobCtx<'_>, with_selection: bool, ) -> error::Result<()> { + p.refresh_profile(descriptor, &job.id, &job.workspace_id, conn) + .await?; let mut cmd = dbt_command(p, &[command]); // The console stays human-readable and goes straight to the job log; the // machine-readable copy goes to a file the progress reporter tails, so @@ -2212,6 +2582,10 @@ async fn run_dbt( .args(["--log-format-file", "json"]) .args(["--log-level-file", p.engine.engine.progress_log_level()]); + if inv.deferral.is_some() { + cmd.args(defer_flags(command, p.engine.engine)); + } + if with_selection && command != "retry" { add_selection(&mut cmd, descriptor, inv)?; } @@ -2232,8 +2606,7 @@ async fn run_dbt( if let Some(t) = descriptor.threads { cmd.args(["--threads", &t.to_string()]); } - let full_refresh = arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh); - if full_refresh && command != "test" { + if full_refresh(descriptor, inv, command)? { cmd.arg("--full-refresh"); } } @@ -2862,7 +3235,11 @@ async fn run_show( `@`) or wildcard (`*`), resolves to a set: run `build` with it instead" ))); } + p.refresh_profile(descriptor, job_id, w_id, conn).await?; let mut cmd = dbt_command(p, &["show"]); + if inv.deferral.is_some() { + cmd.args(defer_flags("show", p.engine.engine)); + } add_vars(&mut cmd, descriptor, inv)?; // Intersected with `resource_type:model`, because `show` is only read-only // for models: dbt dispatches a selected SEED through its seed runner and @@ -2882,7 +3259,8 @@ async fn run_show( conn, SHOW_MAX_OUTPUT_BYTES, ) - .await?; + .await? + .stdout; // dbt frames the rows as `{"node": …, "show": [ … ]}`, pretty-printed, with a // banner before and a deprecation summary after — so neither "the line starting // with `{`" nor "first `{` to the end" parses. A streaming deserializer stops at @@ -2956,6 +3334,7 @@ fn build_result( totals, nodes, invocation_args: inv.raw_args.clone(), + deferred_to: inv.deferral.as_ref().map(|d| d.published_by), } } @@ -3064,7 +3443,7 @@ async fn run_parse_only( // manifest and the selection while the warehouse only keys them — so a project // with no warehouse identity still reports what dbt found. The placeholder // reaches no row: the guard below returns before anything is written. - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, p.warehouse.as_deref().unwrap_or("unkeyed"), p.default_database.as_deref(), @@ -3084,6 +3463,20 @@ async fn run_parse_only( else { return Ok(to_raw_value(&result)); }; + // AFTER the guard: the pass is a second `dbt compile` and a parquet decode, + // and a parse that stores nothing has nowhere to put what it would produce. + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + "parse", + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; match conn { Connection::Sql(db) => match job.runnable_id.map(|h| h.0) { Some(script_hash) => { @@ -3143,11 +3536,70 @@ async fn run_parse_only( Ok(to_raw_value(&result)) } +/// Fold this project's column lineage into the graph about to be stored, when +/// the descriptor asked for it. +/// +/// One helper for all three ingests — deploy, editor parse, per-run refresh — +/// because a graph that carries column lineage in one provenance and not another +/// reads as the lineage having disappeared. +async fn attach_column_index( + ingested: &mut windmill_common::dbt_manifest::IngestedManifest, + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result<()> { + // The nodes this graph kept, so the pass reads only rows it could store: the + // index describes the whole project, this graph one selection of it. + let kept: std::collections::HashSet<&str> = ingested + .nodes + .iter() + .map(|n| n.unique_id.as_str()) + .collect(); + let index = + crate::dbt_column_index::collect( + p, descriptor, inv, command, ctx, job_id, w_id, conn, &kept, + ) + .await?; + drop(kept); + let Some(index) = index else { + return Ok(()); + }; + let found = index.edges.len(); + ingested.attach_column_index(index); + let kept = ingested.column_edges.len(); + let typed: usize = ingested + .nodes + .iter() + .filter(|n| n.column_schema.is_some()) + .count(); + // Counted here rather than at the pass: the index describes the whole + // project and this graph describes one selection of it, so `found` is what + // dbt produced and `kept` is what the graph can draw. + let dropped = match found.saturating_sub(kept) { + 0 => String::new(), + n => format!(" ({n} outside this graph or past the cap)"), + }; + append_logs( + job_id, + w_id, + format!("\nIngested {kept} column lineage edges{dropped} and typed {typed} nodes\n"), + conn, + ) + .await; + Ok(()) +} + /// Refresh the stored graph from the manifest this run produced. async fn ingest_from_run( p: &PreparedProject, descriptor: &DbtDescriptor, inv: &Invocation, + command: &str, ctx: &mut JobCtx<'_>, job: &MiniPulledJob, conn: &Connection, @@ -3164,12 +3616,24 @@ async fn ingest_from_run( // filter this run's manifest by a different node set than it built. let selected = resolve_selection(p, descriptor, inv, ctx, &job.id, &job.workspace_id, conn).await?; - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, p.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + command, + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; // Only a run whose models are its own snapshots per run. A static // descriptor at a moved profile re-ingests the VERSION's graph, since the // move outlives the run; one that neither drifted nor overrode anything @@ -3177,7 +3641,7 @@ async fn ingest_from_run( let snapshot_job = p.graph_refresh.snapshot_job(job.id); match conn { Connection::Sql(db) => { - persist_ingest( + let published = persist_ingest( db, &job.workspace_id, script_path, @@ -3190,6 +3654,13 @@ async fn ingest_from_run( p.graph_refresh.publishes_ownership(), ) .await?; + // Publishing ownership from a RUN makes this project the owner of + // those relations exactly as a deploy does, so it can be what leaves + // a subscription accepted while the relation had no producer with dbt + // as its only one. Same warning the deploy emits. + if published && p.graph_refresh.publishes_ownership() { + warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await; + } } // An agent worker reaches these tables only through the API. Publishing // is the whole of what it needs: a worker that can replace the graph @@ -3242,7 +3713,7 @@ enum GraphPublisher { /// Replace this script's graph, unless a newer version of it has been deployed. /// /// Write one ingest: the sidecar rows and the `asset` usages the manifest -/// implies. No subscriptions — a `dbt://` one could never fire. +/// implies. No subscriptions — a dbt project is not woken by the cascade. /// /// Returns whether this job was still the one entitled to the path-keyed half — /// false once a newer version has superseded it, or once the version is gone. @@ -3333,9 +3804,10 @@ async fn persist_ingest( &ingested.assets, ) .await?; - // A `dbt://` subscription can never fire, so none are derived from the - // manifest. The delete stays to clear what earlier versions wrote, which would - // otherwise keep drawing cascade arrows that wake nothing. + // A dbt project is not woken by the asset cascade (refused at deploy), so + // none are derived from the manifest either. The delete stays to clear what + // earlier versions wrote, which would otherwise keep drawing cascade arrows + // that wake nothing. sqlx::query!( "DELETE FROM script_trigger WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2 @@ -3349,6 +3821,56 @@ async fn persist_ingest( Ok(true) } +/// Log the `// on dbt://…` subscriptions this project's relations leave dormant. +/// +/// Subscribing to a relation dbt already owns is refused at the subscriber's +/// deploy, but one deployed while nothing produced that relation is accepted — +/// as it is for every other asset kind — and an ingest is what can afterwards +/// make dbt its only producer. A dbt run does not dispatch, so such an edge is +/// drawn on the canvas and never fires; a job's own log is where that ordering is +/// visible. +/// +/// Called from both points that publish ownership, the deploy and a run whose +/// static descriptor found its profile moved — either can be the one that claims +/// the relation. +async fn warn_dormant_subscribers( + db: &sqlx::Pool, + w_id: &str, + job_id: &Uuid, + ingested: &windmill_common::dbt_manifest::IngestedManifest, + conn: &Connection, +) { + use windmill_common::assets::AssetUsageAccessType; + let relations: Vec = ingested + .assets + .iter() + .filter(|a| { + matches!( + a.access_type.or(a.alt_access_type), + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) + }) + .map(|a| a.path.clone()) + .collect(); + match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await { + Ok(edges) if !edges.is_empty() => { + append_logs( + job_id, + w_id, + format!( + "\nThese subscriptions will not fire — a dbt run does not trigger downstream \ + runs, and nothing else writes their relation:\n {}\n", + edges.join("\n ") + ), + conn, + ) + .await; + } + Ok(_) => {} + Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"), + } +} + /// Serialize publishers for one script path and confirm this job's version is /// still the newest. Both happen inside the caller's transaction, so a newer /// publisher either commits before this check sees it, or waits behind it and @@ -3411,6 +3933,12 @@ async fn resolve_selection( return Ok(None); } let mut cmd = dbt_command(p, &["ls"]); + // The same state the build resolves through, or a `result:` selector — which + // reads `run_results.json` out of it, and which `select` passes to dbt + // verbatim — fails here, before the build that would have honoured it. + if inv.deferral.is_some() { + cmd.args(defer_flags("ls", p.engine.engine)); + } // A project whose models call `var()` without a default fails to parse // without these, so the selection resolver needs them exactly as the run // does. Placeholders that only a run can fill are dropped rather than @@ -3424,11 +3952,15 @@ async fn resolve_selection( } cmd.args(["--output", "json", "--quiet"]); add_selection(&mut cmd, descriptor, inv)?; + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; // Captured directly, not through `handle_child`: its `pipe_stdout` path goes // through the job-log writer, which `NO_LOGS_AT_ALL` discards — the selection // would resolve to the empty set and the ingest would wipe the script's assets // while dbt went on building the descriptor's models. - let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES).await?; + let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES) + .await? + .stdout; let mut set = std::collections::HashSet::new(); for line in stdout.lines() { let line = line.trim(); @@ -3441,15 +3973,37 @@ async fn resolve_selection( } } } - if set.is_empty() { - // A selection that matches nothing would be ingested as "this script - // owns no relations", wiping its graph and cascade edges — the same - // outcome a failed capture produces, and indistinguishable from it. - // Refuse rather than silently un-wire the script. + // Empty is a real answer from a `state:` or `result:` method and from nothing + // else: `state:modified+` matches nothing exactly when nothing changed since + // the published state, and a run with no work to do is a successful one. Any + // other selection matching nothing is a selector that names nothing — a + // misspelled model, say — which must not pass as a build that did its job. + // Exempting by ORIGIN rather than by method would let every such typo through. + // + // What makes the exemption safe is that the empty set is never ingested as + // ownership, and that now holds through `check_state_selectors`: a `state:` + // or `result:` method survives it only from a run's OWN selection, which + // makes `add_caller_args` set `per_run_models`, which makes + // `publishes_ownership()` false, so the run stores a snapshot of its own. + // Relax the descriptor arm there and a descriptor-narrowed `state:modified+` + // reaches here on an unchanged project and wipes the graph the `else` below + // guards, with nothing failing. + if set.is_empty() && !selection_names(&select, &exclude, &["state", "result"]) { return Err(Error::ExecutionErr( - "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection rather \ - than deploying a script that owns nothing" - .to_string(), + if selection_is_overridden(descriptor, &inv.args)? { + "this run's `select`/`exclude` matched no dbt nodes, so it would build nothing; \ + check the selector. Only a `state:` or `result:` selector may match nothing, \ + its empty answer being a real one" + .to_string() + } else { + // The descriptor's is also ingested as "this script owns no + // relations", wiping its graph and cascade edges — the same + // outcome a failed capture produces, and indistinguishable from + // it. Refuse rather than silently un-wire the script. + "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection \ + rather than deploying a script that owns nothing" + .to_string() + }, )); } Ok(Some(set)) @@ -3459,6 +4013,34 @@ async fn resolve_selection( /// what is kept is the TAIL, because dbt prints its error summary last. const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; +/// What a captured invocation produced. `stderr` is where dbt writes its +/// diagnostics — the errors and warnings block — so a caller that has to explain +/// a SUCCESSFUL run needs it as much as a failing one does. +pub(crate) struct Captured { + pub stdout: String, + pub stderr: String, + /// Whether the child exited zero. Separate from the `Result` on purpose: an + /// `Err` from `run_captured` is the JOB's — a cancellation or its deadline — + /// so a caller that tolerates a failed command must still propagate one. + pub success: bool, + /// Whether the output ceiling cut the child short. Only ever true under + /// [`Overflow::Truncate`]. + pub truncated: bool, +} + +/// What an over-long stdout means to the caller. +/// +/// The ceiling belongs to the PASS, not to the job: a caller that only annotates +/// a job wants to keep what it read and carry on, while one whose whole result +/// is that output has nothing to return without it. +#[derive(PartialEq, Eq, Clone, Copy)] +pub(crate) enum Overflow { + /// Fail the job. For a command whose output IS the answer. + Fail, + /// Stop reading, kill the child, and report `truncated`. + Truncate, +} + /// Run a command for its stdout under the job's cancellation and timeout. /// The same poller `handle_child` uses drives them, so a cancel or a deadline /// drops the wait future — which owns the child, and `kill_on_drop` then @@ -3471,7 +4053,7 @@ const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; /// never holds more than it, so it has to be enforced while reading. Both pipes /// are drained concurrently because a child that fills the one nobody reads /// blocks forever. -async fn run_capturing( +pub(crate) async fn run_captured( mut cmd: Command, name: &str, ctx: &mut JobCtx<'_>, @@ -3479,7 +4061,8 @@ async fn run_capturing( w_id: &str, conn: &Connection, max_stdout_bytes: usize, -) -> error::Result { + on_overflow: Overflow, +) -> error::Result { use tokio::io::AsyncReadExt; let mut child = cmd @@ -3514,6 +4097,7 @@ async fn run_capturing( let mut out_buf = vec![0u8; 16 * 1024]; let mut err_buf = vec![0u8; 16 * 1024]; let (mut out_open, mut err_open) = (true, true); + let mut truncated = false; while out_open || err_open { tokio::select! { r = stdout_pipe.read(&mut out_buf[..]), if out_open => match r { @@ -3521,14 +4105,19 @@ async fn run_capturing( Ok(n) => { if stdout.len() + n > max_stdout_bytes { // Killed here rather than left to `kill_on_drop` - // so the child is gone before the error unwinds, - // not merely once this future is dropped. + // so the child is gone before this returns, not + // merely once the future is dropped. let _ = child.kill().await; - return Err(Error::ExecutionErr(format!( - "{name} produced more than {} MB of output. Narrow the \ - selection, or query the relation from a SQL script.", - max_stdout_bytes / 1024 / 1024 - ))); + if on_overflow == Overflow::Fail { + return Err(Error::ExecutionErr(format!( + "{name} produced more than {} MB of output. Narrow the \ + selection, or query the relation from a SQL script.", + max_stdout_bytes / 1024 / 1024 + ))); + } + truncated = true; + out_open = false; + continue; } stdout.extend_from_slice(&out_buf[..n]); } @@ -3551,7 +4140,7 @@ async fn run_capturing( .wait() .await .map_err(|e| Error::internal_err(format!("{name} failed: {e}")))?; - Ok((status, stdout, stderr)) + Ok((status, stdout, stderr, truncated)) }, ctx.worker_name, w_id, @@ -3561,14 +4150,46 @@ async fn run_capturing( })), ) .await?; - let (status, stdout, stderr) = out; - if !status.success() { + let (status, stdout, stderr, truncated) = out; + Ok(Captured { + stdout: String::from_utf8_lossy(&stdout).to_string(), + stderr: String::from_utf8_lossy(&stderr).to_string(), + // A killed child reports failure; under `Truncate` that is the ceiling's + // doing, not the project's, and the caller reads `truncated` to tell. + success: status.success(), + truncated, + }) +} + +/// `run_captured`, with a non-zero exit folded into the error — what a caller +/// that needs the command to have WORKED wants. +pub(crate) async fn run_capturing( + cmd: Command, + name: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + max_stdout_bytes: usize, +) -> error::Result { + let captured = run_captured( + cmd, + name, + ctx, + job_id, + w_id, + conn, + max_stdout_bytes, + Overflow::Fail, + ) + .await?; + if !captured.success { return Err(Error::ExecutionErr(format!( "{name} failed: {}", - String::from_utf8_lossy(&stderr) + captured.stderr ))); } - Ok(String::from_utf8_lossy(&stdout).to_string()) + Ok(captured) } /// Run a preparation command through the same child handler the build uses, so @@ -3761,7 +4382,7 @@ async fn save_run_state( if let Connection::Sql(db) = conn { { // Only while a live dbt version stays at this path — the test - // `clear_dbt_run_state_if_path_retired` retires state by, plus the + // `clear_dbt_script_state_if_path_retired` retires state by, plus the // language, since a rename leaves the old path archived rather than // deleted and a path can come back as another language. A job already // running finishes after those move or clear the row: writing then @@ -3923,6 +4544,12 @@ pub struct Invocation { /// what it pointed at must not. pub raw_args: HashMap>, pub envs: HashMap, + /// The stored dbt state this invocation resolves an unbuilt `ref()` through, + /// materialised into the job directory. Carried here rather than passed to + /// each phase: the model phase, the `after_all` tests and every in-job node + /// retry must all resolve a `ref()` the same way, or the tests assert against + /// relations the models never read. + pub deferral: Option, /// A run must fail on a `{{ }}` placeholder it cannot fill; a deploy, which /// has no arguments at all, tolerates them. Declared rather than inferred /// from the argument count: a run submitted with `{}` is still a run, and @@ -4075,11 +4702,12 @@ async fn restore_from_db( if !has_retryable_node(&row.run_results) { return Err(nothing_to_retry()); } - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - tokio::fs::write(target.join("run_results.json"), &row.run_results) - .await - .map_err(|e| Error::internal_err(format!("restoring run_results.json: {e}")))?; + write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&row.run_results), + StateManifest::None, + ) + .await?; // No manifest came with the row, so one has to be re-derived — but not here: // these arguments are as SUBMITTED, and a `$var:` in them shapes the graph // only once resolved. The caller resolves, then parses. @@ -4311,20 +4939,16 @@ async fn restore_run_state( return Err(different_project()); } let saved_args_digest = saved_args_digest.map(str::to_string); - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - // From the bytes already read, not by copying the file again: a burst of saves - // can prune this generation mid-restore, and a `dbt retry` whose + // The results go from the bytes already read, not by copying the file again: a + // burst of saves can prune this generation mid-restore, and a `dbt retry` whose // `run_results.json` went missing rebuilds nothing and reports success. The // manifest has no such copy, so a failure there falls back to a `dbt parse`. - tokio::fs::write(target.join("run_results.json"), &saved_results) - .await - .map_err(|e| { - Error::internal_err(format!("could not restore the previous run's results: {e}")) - })?; - let needs_parse = tokio::fs::copy(snapshot.join("manifest.json"), target.join("manifest.json")) - .await - .is_err(); + let needs_parse = !write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&saved_results), + StateManifest::CopyOf(snapshot.join("manifest.json")), + ) + .await?; // The generation was chosen from a row read before the file work above. A run // finishing in that window publishes a newer one, and resuming the superseded // generation redoes nodes it has already rebuilt — appending to an incremental @@ -4518,7 +5142,11 @@ fn has_retryable_node(run_results: &str) -> bool { } /// Append `--vars` if the descriptor (or the run) declares any. -fn add_vars(cmd: &mut Command, descriptor: &DbtDescriptor, inv: &Invocation) -> error::Result<()> { +pub(crate) fn add_vars( + cmd: &mut Command, + descriptor: &DbtDescriptor, + inv: &Invocation, +) -> error::Result<()> { let vars = resolved_vars(descriptor, &inv.args, inv.strict)?; if !vars.is_empty() { cmd.args(["--vars", &serde_json::to_string(&vars).unwrap_or_default()]); @@ -4713,10 +5341,26 @@ fn add_selection( descriptor: &DbtDescriptor, inv: &Invocation, ) -> error::Result<()> { - for s in effective_select(descriptor, inv)? { + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; + // The seam itself, which the DEPLOY reaches without going through a run: it + // resolves the descriptor's selection to decide what the script owns, and + // never computes a `defer`. A run has been checked earlier, where the message + // can still come before the state fetch. + check_state_selectors( + &select, + &exclude, + if inv.deferral.is_some() { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(descriptor, &inv.args)?, + )?; + for s in select { cmd.args(["--select", &s]); } - for s in effective_exclude(descriptor, inv)? { + for s in exclude { cmd.args(["--exclude", &s]); } if let Some(sel) = effective_selector(descriptor, inv)? { @@ -4725,6 +5369,115 @@ fn add_selection( Ok(()) } +/// The method a selection token names, with the graph operators that can +/// surround a node stripped (`@model`, `+model`, `2+model`, `model+`). +fn selector_method(token: &str) -> Option<&str> { + token + .trim_start_matches('@') + .trim_start_matches(|c: char| c.is_ascii_digit()) + .trim_start_matches('+') + .split_once(':') + .map(|(method, _)| method) +} + +/// Every method a selection names. Each entry is a union of whitespace-separated +/// tokens, and each of those an intersection of comma-separated ones. +fn selection_methods<'a>(entries: &'a [String]) -> impl Iterator { + entries + .iter() + .flat_map(|entry| entry.split([' ', '\t', ','])) + .filter_map(selector_method) +} + +/// Whether the run being checked has the state directory a `state:` or `result:` +/// method reads, or could be given one. +#[derive(Clone, Copy)] +enum StateAccess<'a> { + /// Deferring, so the directory is there. + Given, + /// Not deferring, and `defer` is what would hand it one. + OnRequest, + /// This command resolves a selection without ever deferring, so no setting + /// gives it a state and "turn `defer` on" would be advice that leads nowhere. + Never(&'a str), +} + +/// Whether a selection names any of these methods. +fn selection_names(select: &[String], exclude: &[String], methods: &[&str]) -> bool { + selection_methods(select) + .chain(selection_methods(exclude)) + .any(|method| methods.contains(&method)) +} + +/// Refuse a selection dbt cannot resolve, before it silently resolves to the +/// wrong thing. +/// +/// `state:` and `result:` compare against the artifacts in `--state`, which only +/// a deferring run is given. The engines do not agree on what happens without +/// one: dbt-core 1.x raises, but dbt-sa-cli and fusion read a missing state as an +/// EMPTY one and exit 0, so `state:modified` builds nothing and `state:new` +/// builds the whole project, each as a run that reports success. +/// +/// From the DESCRIPTOR they are refused whether or not the run defers, because +/// that selection also decides which nodes the script owns, and "whatever changed +/// last" is not an ownership answer — the deploy resolves it with no state at all. +/// They describe one run, so they belong in a run's own `select`. +/// +/// `source_status:` compares `sources.json`, which `dbt source freshness` writes +/// and no run publishes here, so it has nothing to compare against under any +/// setting. +/// +/// Only what `select` and `exclude` spell directly: a method reached through a +/// `selectors.yml` definition is named nowhere the worker can read, and dbt's +/// own behaviour is what stands there. +fn check_state_selectors( + select: &[String], + exclude: &[String], + access: StateAccess<'_>, + from_descriptor: bool, +) -> error::Result<()> { + for method in selection_methods(select).chain(selection_methods(exclude)) { + match method { + "source_status" => { + return Err(Error::BadRequest( + "a `source_status:` selector compares the source freshness recorded in \ + `sources.json`, which `dbt source freshness` writes and no run stores \ + here, so there is nothing for it to compare against. Drop the selector" + .to_string(), + )) + } + "state" | "result" if from_descriptor => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector describes what ONE run builds, but the descriptor's \ + selection also decides which nodes this script owns, which a deploy \ + resolves with no state to compare against. Move it to the `select` of a \ + run with `defer` on" + ))) + } + "state" | "result" => match access { + StateAccess::Given => {} + StateAccess::OnRequest => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and only a run with `defer` on is given \ + that state. Turn `defer` on, or drop the selector" + ))) + } + StateAccess::Never(command) => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and `{command}` resolves its selection \ + without building and never defers, so no setting hands it that state. \ + Drop the selector" + ))) + } + }, + _ => {} + } + } + Ok(()) +} + /// Whether this invocation chose its own `select`/`exclude`. /// /// DIFFERENT from the descriptor's, not merely present: `parse_dbt_sig` gives @@ -4748,6 +5501,41 @@ fn selection_is_overridden( Ok(differs("select", &descriptor.select)? || differs("exclude", &descriptor.exclude)?) } +/// Whether this invocation rebuilds incremental models from scratch: the run +/// form's answer when it gave one, else the descriptor's — and never for a +/// `test`, which builds nothing whatever the form said. +/// +/// Shared with the column-lineage pass rather than recomputed there, because +/// `is_incremental()` branches on it: the same model compiles to different SQL — +/// a `{{ this }}` self-join, and any `ref()` inside the incremental branch — so a +/// pass that guessed would describe a build that never ran. +/// +/// `test` returns false because `dbt test` rejects `--full-refresh` outright. +/// It never arrives as a caller's `dbt_command` — the allowlist has no such +/// value — so reading only that allowlist suggests this branch is dead. It is +/// not: `run_dbt` is invoked with `"test"` directly for the `after_all` test +/// phase, and an `after_all` project with `full_refresh: true` reaches here. +pub(crate) fn full_refresh( + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, +) -> error::Result { + if command == "test" { + return Ok(false); + } + Ok(arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh)) +} + +/// Whether this run answered `full_refresh` differently from the deployed +/// descriptor. Like a selection override it changes what the graph describes, +/// since an incremental branch can carry its own `ref()`. +fn full_refresh_is_overridden( + descriptor: &DbtDescriptor, + args: &HashMap>, +) -> error::Result { + Ok(arg_bool(args, "full_refresh")?.is_some_and(|v| v != descriptor.full_refresh)) +} + /// The descriptor's named selector, unless this run named its own selection. /// /// dbt resolves `--selector` INSTEAD of `--select`, so passing both makes the @@ -5007,20 +5795,47 @@ mod tests { // and without normalizing it the saved run is never recognized as its own. #[test] fn profile_identity_ignores_the_attempts_token() { - let yaml = |tok: &str| format!("host: \"wh\"\npassword: \"{tok}\"\n"); + let yaml = |v: &str| format!("host: \"{v}\"\nuser: \"u\"\n"); let dir = Path::new("/tmp/windmill/w/job-1/profiles"); assert_eq!( profile_identity_digest(&yaml("tok-first"), dir, None, "tok-first"), profile_identity_digest(&yaml("tok-retry"), dir, None, "tok-retry") ); - // A password that is NOT the job's token is the connection, and changing - // it must still read as a different warehouse. + // A value that is NOT the job's token is the connection, and changing it + // must still read as a different warehouse. assert_ne!( profile_identity_digest(&yaml("static-a"), dir, None, "tok-first"), profile_identity_digest(&yaml("static-b"), dir, None, "tok-retry") ); } + // Project code can leave a symlink where the worker later writes the profile: + // either the directory or the file in it. Neither may be followed. + #[cfg(unix)] + #[tokio::test] + async fn fresh_dir_never_follows_what_the_jail_left() { + let root = tempfile::tempdir().unwrap(); + let host = root.path().join("host"); + std::fs::create_dir(&host).unwrap(); + std::fs::write(host.join("profiles.yml"), "host file").unwrap(); + let dir = root.path().join("dbt_profiles"); + + std::os::unix::fs::symlink(&host, &dir).unwrap(); + fresh_dir(&dir).await.unwrap(); + assert!(!std::fs::symlink_metadata(&dir).unwrap().is_symlink()); + std::fs::write(dir.join("profiles.yml"), "rendered").unwrap(); + + fresh_dir(&dir).await.unwrap(); + std::os::unix::fs::symlink(host.join("profiles.yml"), dir.join("profiles.yml")).unwrap(); + fresh_dir(&dir).await.unwrap(); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); + + assert_eq!( + std::fs::read_to_string(host.join("profiles.yml")).unwrap(), + "host file" + ); + } + // The jail profile is protobuf text format, and the project path and the // descriptor's environment land inside string literals. An unescaped quote or // newline closes the literal and lets the rest be read as further directives — @@ -5738,6 +6553,79 @@ mod tests { .unwrap(); assert!(untouched.publishes_ownership()); assert_eq!(untouched.snapshot_job(job), None); + + // `resolve_selection` lets a selection match nothing on exactly this + // predicate, because a run that scoped its own selection stores a + // snapshot instead of publishing ownership. Should the two ever drift + // apart, an empty caller selection would wipe the script's graph and + // cascade edges, which is the outcome that guard exists to prevent. + // One-directional: a `vars` override also withholds ownership without + // touching the selection, which is why this is an implication and not an + // equivalence. + for args in [ + arg("select", r#"["state:modified+"]"#), + arg("exclude", r#"["tag:nightly"]"#), + ] { + assert!(selection_is_overridden(&descriptor, &args).unwrap()); + let mut g = GraphRefresh::default(); + g.add_caller_args(&descriptor, &args).unwrap(); + assert!( + !g.publishes_ownership(), + "an overridden selection must not publish ownership" + ); + } + + // `full_refresh` decides whether `is_incremental()` is true, so an + // incremental model's self-join — and any `ref()` inside that branch — + // exists in one answer and not the other. A run that flips it describes + // a different graph, and gets its own. + let mut refreshed = GraphRefresh::default(); + refreshed + .add_caller_args(&descriptor, &arg("full_refresh", "true")) + .unwrap(); + assert!(refreshed.needed()); + assert_eq!(refreshed.snapshot_job(job), Some(job)); + + // The same echo rule: the form posts the descriptor's own value back on + // every run, and reading that as an override would make each one + // caller-scoped. + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let mut echoed_flag = GraphRefresh { profile_drift: true, ..Default::default() }; + echoed_flag + .add_caller_args(&always, &arg("full_refresh", "true")) + .unwrap(); + assert_eq!(echoed_flag.snapshot_job(job), None); + } + + /// The build and the analysis pass read this through one function, so they + /// cannot disagree about which SQL the run compiles — including for `test`, + /// which rebuilds nothing whatever the descriptor or the form said. + #[test] + fn full_refresh_is_one_answer_for_the_build_and_the_pass() { + let inv = |args: HashMap>| Invocation { + args, + raw_args: Default::default(), + envs: Default::default(), + strict: true, + deferral: None, + }; + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let never = DbtDescriptor::default(); + let on = HashMap::from([( + "full_refresh".to_string(), + RawValue::from_string("true".to_string()).unwrap(), + )]); + + assert!(full_refresh(&always, &inv(Default::default()), "build").unwrap()); + assert!(!full_refresh(&never, &inv(Default::default()), "build").unwrap()); + assert!( + full_refresh(&never, &inv(on), "build").unwrap(), + "the form's answer wins over the descriptor's" + ); + assert!( + !full_refresh(&always, &inv(Default::default()), "test").unwrap(), + "a test builds nothing, so neither the build nor the pass may pass the flag" + ); } // `dbt retry` restores the previous run's target/ from this directory, so two @@ -5766,6 +6654,128 @@ mod tests { ); } + // A profile whose location dbt renders cannot be told apart from another + // rendering of itself, so it neither publishes state nor defers. Both + // delimiters count: a conditional block moves a schema exactly as an + // `env_var()` substitution does. + #[test] + fn a_rendered_profile_location_is_recognised_by_either_delimiter() { + assert!(is_jinja("{{ env_var('DBT_SCHEMA') }}")); + assert!(is_jinja( + "{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}" + )); + assert!(!is_jinja("analytics")); + assert!(!is_jinja("")); + } + + // dbt-sa-cli and fusion exit 0 on a state selector with no state, so nothing + // downstream would report this: the graph operators have to be stripped for + // the method to be seen at all. + #[test] + fn a_state_selector_is_found_under_any_graph_operator() { + // A run's own selection, which is the only place these belong. + let refused = |sel: &str, access: StateAccess<'_>| { + check_state_selectors(&[sel.to_string()], &[], access, false).is_err() + }; + for sel in [ + "state:modified", + "state:modified+", + "+state:new", + "@state:modified", + "2+state:modified+3", + "tag:nightly,state:modified", + "stg_orders+ result:error+", + ] { + assert!( + refused(sel, StateAccess::OnRequest), + "{sel} should need `defer`" + ); + assert!( + !refused(sel, StateAccess::Given), + "{sel} should pass while deferring" + ); + // A parse resolves a selection without ever deferring, so it is + // refused where a run would have been told to turn `defer` on. + assert!( + refused(sel, StateAccess::Never("parse")), + "{sel} cannot parse" + ); + // The descriptor's selection also decides what the script owns, and + // the deploy resolves it with no state, so deferring cannot save it. + assert!( + check_state_selectors(&[sel.to_string()], &[], StateAccess::Given, true).is_err(), + "{sel} should never be a descriptor selection" + ); + } + // A node whose name merely starts with a method's letters is not one. + for sel in ["stg_orders+", "tag:nightly", "stateful_model+"] { + assert!( + !refused(sel, StateAccess::OnRequest), + "{sel} is not a state selector" + ); + } + // No run publishes `sources.json`, so deferring does not help. + for access in [ + StateAccess::Given, + StateAccess::OnRequest, + StateAccess::Never("parse"), + ] { + assert!(refused("source_status:fresher+", access)); + } + // `exclude` reaches dbt the same way `select` does. + assert!(check_state_selectors( + &[], + &["state:modified".to_string()], + StateAccess::OnRequest, + false + ) + .is_err()); + + // The same recognition decides which empty selections `resolve_selection` + // lets through. Only these two answer "nothing" meaningfully; a selector + // naming nothing must not pass as a build that did its work. + const STATE_BACKED: &[&str] = &["state", "result"]; + for sel in ["state:modified+", "result:error+", "tag:x,state:new"] { + assert!( + selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + for sel in [ + "mispelled_model", + "tag:nightly", + "stg_orders+", + "source_status:fresher+", + ] { + assert!( + !selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + } + + // The one flag choice that is silently wrong rather than loudly wrong: a + // `retry` handed `--state` resumes the SUCCESSFUL run stored there and + // rebuilds nothing, reporting a green retry of a failed run. + #[test] + fn a_retry_is_never_handed_the_deferral_as_its_state() { + assert_eq!( + defer_flags("build", DbtEngine::DbtCore1x), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("test", DbtEngine::Fusion), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("retry", DbtEngine::DbtCore1x), + ["--defer-state", crate::dbt_state::STATE_DIR] + ); + for engine in [DbtEngine::DbtCore2x, DbtEngine::Fusion] { + assert!(defer_flags("retry", engine).is_empty()); + } + } + #[test] fn events_without_a_relation_are_not_materializations() { // A test node has no relation of its own. diff --git a/backend/windmill-worker/src/dbt_profiles.rs b/backend/windmill-worker/src/dbt_profiles.rs index 3d96f84676..f81aea2ae5 100644 --- a/backend/windmill-worker/src/dbt_profiles.rs +++ b/backend/windmill-worker/src/dbt_profiles.rs @@ -7,12 +7,42 @@ use serde_json::Value; use windmill_common::error::{self, Error}; +use windmill_parser_yaml::dbt::DbtEngine; /// Written beside `profiles.yml`, and named absolutely in `sslrootcert`: dbt /// runs with the project as its working directory and hands the path to the /// driver unchanged. pub const ROOT_CERT_FILENAME: &str = "server-ca.pem"; +/// What a credential's value becomes in [`RenderedProfile::identity`]. +const MASKED_CREDENTIAL: &str = "$CREDENTIAL"; + +/// Whether a target key holds a credential rather than part of the address. By +/// name, because a `dbt_profile` block's keys are its adapter's own: `password`, +/// dbt-postgres's `pass`, `token`, `private_key_passphrase`, `client_secret`, +/// `aws_secret_access_key` and the `key_id` rotated with it, … An endpoint is +/// address even when its name says otherwise: BigQuery's `token_uri` is where the +/// token comes from. +fn is_credential_key(key: &str) -> bool { + let key = key.to_ascii_lowercase(); + if key.ends_with("_uri") || key.ends_with("_url") || key.contains("endpoint") { + return false; + } + key == "pass" + || [ + "password", + "passphrase", + "secret", + "token", + "private_key", + "api_key", + "access_key", + "key_id", + ] + .iter() + .any(|c| key.contains(c)) +} + /// The per-adapter facts, so each adapter states them together and a new one /// cannot inherit another's by omission. `PG` is the base every arm spreads /// from: most adapters differ from Postgres only in their name and package. @@ -469,6 +499,10 @@ fn port_of(resource: &Value, default: i64) -> error::Result { #[derive(Debug)] pub struct RenderedProfile { pub yaml: String, + /// `yaml` with every credential's value masked: what run identity hashes. A + /// rotated token or password is still the same connection, and an OAuth token + /// rotates every few minutes, so hashing it would refuse nearly every retry. + pub identity: String, pub schema: Option, pub database: Option, /// A private CA the caller must write next to `profiles.yml`, under the @@ -483,8 +517,11 @@ pub struct RenderedProfile { /// from the descriptor when set, else from the resource, else from dbt's own /// per-adapter default — dbt errors out clearly when it ends up missing, which /// is a better failure than a Windmill-invented default. +#[allow(clippy::too_many_arguments)] pub fn render_profile( adapter: &DbtAdapter, + // The engines read some keys differently; see the Snowflake token below. + engine: DbtEngine, resource: &Value, profile_name: &str, target: &str, @@ -593,11 +630,18 @@ pub fn render_profile( out.push(("user".into(), quoted(&u))); } // Key-pair is Windmill's own snowflake resource shape; the - // `snowflake_oauth` type carries a token instead, which dbt only - // accepts alongside `authenticator: oauth` — without both, the - // profile renders with no credential at all and cannot connect. + // `snowflake_oauth` type carries an access token instead, which needs + // an `authenticator` saying so. dbt-core 1.x reads `oauth` + `token` + // as one. The Rust engines read `oauth` as the refresh-token flow and + // refuse a profile without client credentials, and send the same + // login for `jwt`, which dbt-snowflake has only from 1.9, while the + // 1.x engine still resolves 1.8. if let Some(t) = s(resource, "token").or_else(|| s(resource, "access_token")) { - out.push(("authenticator".into(), quoted("oauth"))); + let authenticator = match engine { + DbtEngine::DbtCore1x => "oauth", + DbtEngine::DbtCore2x | DbtEngine::Fusion => "jwt", + }; + out.push(("authenticator".into(), quoted(authenticator))); out.push(("token".into(), quoted(&t))); } else if let Some(k) = s(resource, "private_key") { out.push(("private_key".into(), quoted(&k))); @@ -612,10 +656,17 @@ pub fn render_profile( out.push((k.into(), quoted(&v))); } } - database = s(resource, "database"); - if let Some(d) = database.clone() { - out.push(("database".into(), quoted(&d))); - } + // dbt-snowflake requires one, and a resource from the OAuth connect flow + // starts without it: naming the field beats dbt's schema error. + let db = s(resource, "database").ok_or_else(|| { + Error::BadRequest( + "a Snowflake target needs a database; add `database` to the warehouse's \ + resource" + .to_string(), + ) + })?; + out.push(("database".into(), quoted(&db))); + database = Some(db); schema = schema.or_else(|| s(resource, "schema")); } KnownAdapter::Bigquery => { @@ -692,24 +743,43 @@ pub fn render_profile( // a newline in one opens a sibling key of the caller's choosing. let (qp, qt) = (yaml_scalar(profile_name), yaml_scalar(target)); let mut yaml = format!("{qp}:\n target: {qt}\n outputs:\n {qt}:\n"); + let mut identity = yaml.clone(); for (k, v) in &out { yaml.push_str(&format!(" {k}: {}\n", v.render())); + let shown = if is_credential_key(k) { + yaml_scalar(MASKED_CREDENTIAL) + } else { + v.render() + }; + identity.push_str(&format!(" {k}: {shown}\n")); } // The service-account document is a nested mapping, not a scalar. if adapter == KnownAdapter::Bigquery { yaml.push_str(" keyfile_json:\n"); + identity.push_str(" keyfile_json:\n"); let obj = resource .as_object() .ok_or_else(|| Error::BadRequest("bigquery resource is not an object".to_string()))?; for (k, v) in obj { if let Some(v) = v.as_str() { yaml.push_str(&format!(" {}: {}\n", yaml_scalar(k), yaml_scalar(v))); + let shown = if is_credential_key(k) { + MASKED_CREDENTIAL + } else { + v + }; + identity.push_str(&format!( + " {}: {}\n", + yaml_scalar(k), + yaml_scalar(shown) + )); } } } Ok(RenderedProfile { yaml, + identity, schema, database, root_certificate_pem: matches!(adapter, KnownAdapter::Postgres) @@ -746,6 +816,7 @@ pub fn render_dbt_profile( " \"type\": {}\n", yaml_scalar(adapter.dbt_type()) )); + let mut identity = yaml.clone(); for (k, v) in block { // A null is an optional field the resource form left unset, and dbt // validates several keys against a schema that rejects one. @@ -762,28 +833,33 @@ pub fn render_dbt_profile( if (k == schema_key && schema_override.is_some()) || (k == "threads" && threads.is_some()) { continue; } - emit_entry(&mut yaml, 6, k, v); + emit_entry(&mut yaml, 6, k, v, false); + emit_entry(&mut identity, 6, k, v, true); } + let mut tail = String::new(); if root_certificate_pem.is_some() { - yaml.push_str(&format!( + tail.push_str(&format!( " \"sslrootcert\": {}\n", yaml_scalar(&profiles_dir.join(ROOT_CERT_FILENAME).to_string_lossy()) )); } if let Some(sc) = schema_override { - yaml.push_str(&format!( + tail.push_str(&format!( " {}: {}\n", yaml_scalar(schema_key), yaml_scalar(sc) )); } if let Some(t) = threads { - yaml.push_str(&format!(" \"threads\": {t}\n")); + tail.push_str(&format!(" \"threads\": {t}\n")); } + yaml.push_str(&tail); + identity.push_str(&tail); let str_key = |k: &str| block.get(k).and_then(|v| v.as_str()).map(|v| v.to_string()); Ok(RenderedProfile { yaml, + identity, schema: schema_override .map(|x| x.to_string()) .or_else(|| str_key(schema_key)), @@ -794,16 +870,21 @@ pub fn render_dbt_profile( /// Emit one target key, nesting as deep as the value goes — an adapter's credential can be /// a mapping (bigquery's `keyfile_json`) or a list. Keys are quoted like values: one nothing -/// here enumerates is as free-form as a password. -fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value) { +/// here enumerates is as free-form as a password. `mask` writes credentials as +/// [`MASKED_CREDENTIAL`], at any depth, for [`RenderedProfile::identity`]. +fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value, mask: bool) { out.push_str(&format!("{}{}:", " ".repeat(indent), yaml_scalar(key))); - emit_value(out, indent, v); + emit_value(out, indent, v, mask, mask && is_credential_key(key)); } /// The value half, after `key:`. An empty collection is emitted INLINE: a block with no /// children reads back as `null`, so `extensions: []` would reach the adapter as a missing /// value rather than the empty list dbt was handed. -fn emit_value(out: &mut String, indent: usize, v: &Value) { +/// +/// `credential` masks scalars only. A collection under a credential-named key is still +/// walked: dbt-duckdb's `secrets:` list holds the endpoint and scope that say which +/// connection it is, each entry judged by its own key. +fn emit_value(out: &mut String, indent: usize, v: &Value, mask: bool, credential: bool) { match v { Value::Object(m) => { // A null is an optional field the resource form left unset, and dbt validates @@ -815,7 +896,7 @@ fn emit_value(out: &mut String, indent: usize, v: &Value) { } out.push('\n'); for (k, v) in kept { - emit_entry(out, indent + 2, k, v); + emit_entry(out, indent + 2, k, v, mask); } } Value::Array(items) => { @@ -828,9 +909,10 @@ fn emit_value(out: &mut String, indent: usize, v: &Value) { for item in items { out.push_str(&pad); out.push('-'); - emit_value(out, indent + 2, item); + emit_value(out, indent + 2, item, mask, credential); } } + _ if credential => out.push_str(&format!(" {}\n", yaml_scalar(MASKED_CREDENTIAL))), _ => out.push_str(&format!(" {}\n", yaml_value(v))), } } @@ -904,6 +986,7 @@ mod tests { "dbname": "warehouse", "sslmode": "require"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -936,6 +1019,7 @@ mod tests { "password": "p", "dbname": "warehouse"}); let p = render_profile( &KnownAdapter::Redshift.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1136,6 +1220,7 @@ mod tests { "http_path": "/sql/1.0/warehouses/x", "token": "t"}); let p = render_profile( &KnownAdapter::Databricks.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1161,6 +1246,7 @@ mod tests { "root_certificate_pem": "-----BEGIN CERTIFICATE-----\nx\n"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1188,6 +1274,7 @@ mod tests { let plain = json!({"host": "h", "dbname": "d", "sslmode": "require"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &plain, "wm", "prod", @@ -1200,29 +1287,107 @@ mod tests { assert_eq!(p.root_certificate_pem, None); } - // `snowflake_oauth` maps to the Snowflake adapter, but its credential is a - // token, which dbt honors only with `authenticator: oauth`. Forwarding neither - // renders a profile with no credential at all. - #[test] - fn snowflake_oauth_renders_its_token() { - let r = json!({"account_identifier": "acc", "username": "u", "token": "tok", - "database": "db", "warehouse": "wh"}); - let p = render_profile( + fn snowflake(engine: DbtEngine, r: &Value) -> error::Result { + render_profile( &KnownAdapter::Snowflake.into(), - &r, + engine, + r, "wm", "prod", None, None, std::path::Path::new("/tmp/p"), ) - .unwrap(); - assert!( - p.yaml.contains(" authenticator: \"oauth\"\n"), - "{}", - p.yaml - ); - assert!(p.yaml.contains(" token: \"tok\"\n")); + } + + // `snowflake_oauth` carries an access token, which only one `authenticator` + // per engine accepts: the Rust engines refuse `oauth` without client + // credentials, and dbt-snowflake before 1.9 has no `jwt`. + #[test] + fn snowflake_oauth_names_its_token_per_engine() { + let r = json!({"account_identifier": "acc", "token": "tok", "database": "db"}); + for (engine, authenticator) in [ + (DbtEngine::DbtCore1x, "oauth"), + (DbtEngine::DbtCore2x, "jwt"), + (DbtEngine::Fusion, "jwt"), + ] { + let p = snowflake(engine, &r).unwrap(); + assert!( + p.yaml + .contains(&format!(" authenticator: \"{authenticator}\"\n")), + "{engine:?}: {}", + p.yaml + ); + assert!(p.yaml.contains(" token: \"tok\"\n")); + } + let err = snowflake( + DbtEngine::DbtCore1x, + &json!({"account_identifier": "acc", "token": "tok"}), + ) + .unwrap_err() + .to_string(); + assert!(err.contains("add `database`"), "{err}"); + } + + // Run identity has to survive a credential rotating, which an OAuth token does + // every few minutes, and still change when the connection moves. + #[test] + fn identity_masks_credentials_but_not_the_connection() { + let rendered = |account: &str, token: &str| { + snowflake( + DbtEngine::DbtCore1x, + &json!({"account_identifier": account, "token": token, "database": "db"}), + ) + .unwrap() + .identity + }; + assert_eq!(rendered("acc", "t1"), rendered("acc", "t2")); + assert_ne!(rendered("acc", "t1"), rendered("other", "t1")); + + // A `dbt_profile` block, whose keys are the adapter's own, nested ones too. + let block = |adapter: KnownAdapter, v: Value| { + render_dbt_profile( + &adapter.into(), + v.as_object().unwrap(), + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap() + .identity + }; + let bq = |project: &str, key: &str, token_uri: &str| { + block( + KnownAdapter::Bigquery, + json!({"type": "bigquery", "project": project, "dataset": "d", + "keyfile_json": {"client_email": "e", "private_key": key, + "token_uri": token_uri}}), + ) + }; + assert_eq!(bq("p", "k1", "t"), bq("p", "k2", "t")); + assert_ne!(bq("p", "k1", "t"), bq("q", "k1", "t")); + assert_ne!(bq("p", "k1", "t"), bq("p", "k1", "elsewhere")); + // Only scalars are masked: a `secrets:` entry still names its endpoint. + // A rotated access key changes its id with its secret. + let duck = |endpoint: &str, secret: &str| { + block( + KnownAdapter::Duckdb, + json!({"type": "duckdb", "path": "x.duckdb", + "secrets": [{"type": "s3", "endpoint": endpoint, + "key_id": format!("id-{secret}"), "secret": secret}]}), + ) + }; + assert_eq!(duck("s3.a", "k1"), duck("s3.a", "k2")); + assert_ne!(duck("s3.a", "k1"), duck("s3.b", "k1")); + let pg = |pass: &str| { + block( + KnownAdapter::Postgres, + json!({"type": "postgres", "host": "h", "user": "u", "pass": pass}), + ) + }; + assert_eq!(pg("p1"), pg("p2")); } // dbt rejects a BigQuery target with no dataset and a service-account JSON @@ -1233,6 +1398,7 @@ mod tests { let r = json!({"project_id": "p", "client_email": "e", "private_key": "k"}); let err = render_profile( &KnownAdapter::Bigquery.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1245,6 +1411,7 @@ mod tests { assert!(err.contains("profile.schema"), "{err}"); let p = render_profile( &KnownAdapter::Bigquery.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1284,6 +1451,7 @@ mod tests { let r = json!({"host": "h", "dbname": "sales", "user": "u"}); let p = render_profile( &KnownAdapter::Mysql.into(), + DbtEngine::DbtCore1x, &r, "wm", "dev", @@ -1304,6 +1472,7 @@ mod tests { fn a_profile_name_or_target_cannot_open_a_sibling_key() { let rendered = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &serde_json::json!({"host": "h", "user": "u", "password": "p", "dbname": "d"}), "prod # hidden", "dev\n evil: yes", @@ -1339,6 +1508,7 @@ mod tests { "password": "p\"\nhost: evil.example.com\n#"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "dev", diff --git a/backend/windmill-worker/src/dbt_state.rs b/backend/windmill-worker/src/dbt_state.rs new file mode 100644 index 0000000000..ecf3d0dbe3 --- /dev/null +++ b/backend/windmill-worker/src/dbt_state.rs @@ -0,0 +1,748 @@ +//! The dbt state a project last built into one environment, and the state +//! directory a run reads it back through. +//! +//! `dbt --defer --state ` resolves a `ref()` the run does not build to the +//! relation the manifest in `` names, instead of to the schema this run +//! writes into. That makes the state a durable, per-environment artifact rather +//! than a cache: the next run of a project usually lands on a worker holding +//! neither the manifest nor the results, so anything worker-local answers for +//! one machine's history rather than for the environment. +//! +//! Two artifacts live in that directory and both are stored: `manifest.json`, +//! which is what a deferral resolves through, and `run_results.json`, which +//! `select`'s `result:` selectors read — and `select` reaches dbt verbatim, so a +//! state directory missing it fails a selection a user may legitimately write. + +use std::path::{Path, PathBuf}; + +use uuid::Uuid; +use windmill_common::error::{self, Error}; +use windmill_common::worker::Connection; + +use crate::dbt_executor::{digest, PreparedProject, ARTIFACTS_DIR}; + +lazy_static::lazy_static! { + /// Above this, an artifact goes to the instance's object storage instead of + /// into the row. A manifest passes a few hundred KB on a handful of models + /// and grows with the project, so this ceiling is what decides whether a + /// large project needs storage configured at all; a small one stays in the + /// database, where it costs no round trip and needs nothing configured. + static ref DBT_STATE_INLINE_MAX_BYTES: usize = std::env::var("DBT_STATE_INLINE_MAX_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8 * 1024 * 1024); +} + +/// The directory `--state` points at. Inside the job directory, so it sits in +/// the sandbox's one writable bind and goes away with the job, and prefixed like +/// the artifacts directory beside it so a project carrying a directory of this +/// name is not overwritten. +/// +/// Passed to dbt RELATIVE, and that is load-bearing rather than tidiness. dbt +/// records the invocation's flags into `run_results.json` and a later +/// `dbt retry` restores them, so an absolute path would name the job directory +/// of the run being resumed — gone by then, leaving the retry to resolve a +/// deferred `ref()` against nothing. Relative, it resolves against the project +/// root, which is whichever job directory the retry landed in. +pub(crate) const STATE_DIR: &str = "wm_dbt_state"; + +/// Where this run's relations live, which is the only thing a deferral is about. +pub(crate) fn environment(p: &PreparedProject) -> String { + environment_key( + p.warehouse.as_deref(), + // The target dbt RUNS, not the descriptor's: it falls back to the + // workspace warehouse's and to the project's own default, so reading the + // descriptor's would put two inherited targets under one empty name. + p.effective_target.as_deref(), + // The pair `relation_root` reports to the graph's drift check, taken + // apart so neither can absorb the other's delimiter below. + p.default_schema.as_deref(), + p.default_database.as_deref(), + ) +} + +/// The warehouse and the target name the environment; the database and schema +/// they resolve to are in the key because a repointed warehouse resource or a +/// moved schema keeps both names while putting the relations somewhere else — +/// and a manifest is a list of relation names, so a deferral has no other way to +/// notice. A move therefore reads as an environment nothing has published yet. +/// +/// Length-prefixed rather than joined on a separator. Every component but the +/// warehouse is spelled by the user — a dbt target name and a schema are both +/// arbitrary strings a profile may quote — so `prod|analytics` + `scratch` and +/// `prod` + `analytics|scratch` would otherwise be one key, and a profile moving +/// between them would read as the same environment rather than as one nothing +/// has published. Same reasoning as `stable_digest`, and still legible in a row: +/// `4:main|4:prod|9:analytics|12:dbt_wh_defer`. What a MESSAGE names is +/// `environment_label`, since this encoding is for storage. +fn environment_key( + warehouse: Option<&str>, + target: Option<&str>, + schema: Option<&str>, + database: Option<&str>, +) -> String { + [warehouse, target, schema, database] + .iter() + .map(|v| { + let v = v.unwrap_or(""); + format!("{}:{v}", v.len()) + }) + .collect::>() + .join("|") +} + +/// The environment as a message names it: the key above is length-prefixed for +/// storage, which is not something to put in front of a caller. +pub(crate) fn environment_label(p: &PreparedProject) -> String { + format!( + "warehouse `{}`, target `{}`, relations in `{}`", + p.warehouse.as_deref().unwrap_or("(none)"), + p.effective_target + .as_deref() + .unwrap_or("(the profile's default)"), + match (p.default_database.as_deref(), p.default_schema.as_deref()) { + (Some(db), Some(schema)) => format!("{db}.{schema}"), + (None, Some(schema)) => schema.to_string(), + _ => "(the adapter's default)".to_string(), + } + ) +} + +/// The state one environment last published. +pub(crate) struct StoredState { + pub manifest: String, + pub run_results: Option, + /// The run that published it, so a deferring run can say what it deferred to. + pub job_id: Uuid, +} + +/// Publish this run's artifacts as the environment's state. +/// +/// Called for a run that BUILT what the script's own descriptor selects and +/// succeeded (see `handle_dbt_job`). Best-effort in the same sense as the retry +/// state: losing it costs the next deferral, not the run that just finished. +/// +/// **What the artifacts may carry follows from that condition.** A publishing run +/// added nothing of its own — no `select` or `vars` override, and a descriptor +/// interpolating a `{{ }}` placeholder into `vars` never publishes at all — so +/// dbt's `run_results.json` records the descriptor's own arguments, which are the +/// script's content. That is why this is keyed by environment where +/// `dbt_run_state` is keyed by principal: the retry state holds whatever a caller +/// submitted, this holds what the script says. Widen the publish condition and +/// that stops being true. +pub(crate) async fn publish( + p: &PreparedProject, + w_id: &str, + job_id: &Uuid, + // The version this job ran. `None` for a preview, which publishes nothing. + script_hash: Option, + // A build recovered by the automatic in-job node retry has a + // `run_results.json` naming only the nodes that retry redid. The manifest is + // unaffected — it is a function of the project, not of what ran — so the + // state is published without results rather than with a set describing some + // other slice of the build. + results_are_partial: bool, + conn: &Connection, +) -> error::Result<()> { + let Connection::Sql(db) = conn else { + // An agent worker reaches the database only through the API, which does + // not expose this table. + return Ok(()); + }; + if p.script_path.is_empty() { + // A preview has no path to key state on, and an empty one would be + // shared by every dbt script in the workspace. + return Ok(()); + } + if p.templated_location { + // Refused on this side too, not only where a deferral reads. A template + // renders to one location per environment while the key sees the + // template, so publishing would file this run's manifest under a key a + // literal profile shares — and de-templating later would make that stale + // manifest readable as the new location's. + return Ok(()); + } + let artifacts = p.project_dir.join(ARTIFACTS_DIR); + // The manifest is what a deferral resolves through, so there is no state + // without one. Every engine writes it beside the results of a build, so this + // is the invocation that built nothing rather than a case to report. + let Ok(manifest) = tokio::fs::read_to_string(artifacts.join("manifest.json")).await else { + return Ok(()); + }; + let run_results = match results_are_partial { + true => None, + false => tokio::fs::read_to_string(artifacts.join("run_results.json")) + .await + .ok(), + }; + let environment = environment(p); + // Uploaded BEFORE the transaction, and to this publication's own keys, so two + // publishers cannot collide on them and nothing here can overwrite an + // artifact a committed row still names. A failure below has only its own + // objects to drop. + let nonce = Uuid::new_v4(); + let (manifest, manifest_key) = store( + manifest, + "manifest.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await?; + let (run_results, run_results_key) = match run_results { + Some(r) => match store( + r, + "run_results.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await + { + Ok(stored) => stored, + Err(e) => { + forget_objects(&[manifest_key, None]).await; + return Err(e); + } + }, + None => (None, None), + }; + let mine = [manifest_key.clone(), run_results_key.clone()]; + // One publisher per environment at a time, so the row and the objects it + // displaces are settled by one of them at a time. An advisory lock rather + // than the row's, because the first publish of an environment has no row to + // lock and is exactly when two runs of a newly deployed script are most + // likely to race. + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + forget_objects(&mine).await; + return Err(e.into()); + } + }; + let staged = async { + sqlx::query_scalar!( + "SELECT pg_advisory_xact_lock($1)", + publication_lock(w_id, &p.script_path, &environment) + ) + .execute(&mut *tx) + .await?; + // The script row FIRST, and held, so a rename, archive or delete of this + // path either waits for this publication or is seen by it. Reading it + // unlocked leaves a window where lifecycle cleanup finds no row to clear, + // finishes, and this transaction then commits state at a path a new + // script goes on to occupy. Script row before sidecar is also the order + // every other dbt writer takes, which is what keeps the two off a + // deadlock. + // + // The version, not just the path: "some live dbt script is here" is also + // satisfied by a script created at a path this one was renamed away from. + // A preview names no version, so `script_hash` is NULL and nothing + // matches — right for a run of content that was never deployed. + let owns_path = sqlx::query_scalar!( + "SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false AND language = 'dbt' + AND (hash = $3 OR $3 = ANY(parent_hashes)) + FOR SHARE", + w_id, + &p.script_path, + script_hash, + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + if !owns_path { + return error::Result::Ok(None); + } + // What the row points at NOW, so those objects can go once this one is + // committed in their place — never before, since a reader that has + // already read the row is about to fetch them. + let displaced = sqlx::query!( + "SELECT manifest_key, run_results_key FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(&mut *tx) + .await? + .map(|r| [r.manifest_key, r.run_results_key]) + .unwrap_or_default() + // Never a key this publication is about to commit. The keys carry a + // per-execution nonce so the two cannot coincide, and this is what says + // so rather than leaving it to be re-derived. + .map(|k| k.filter(|k| !mine.iter().flatten().any(|m| m == k))); + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest, manifest_key, run_results, + run_results_key, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now()) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET + job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest, + manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results, + run_results_key = EXCLUDED.run_results_key, updated_at = now()", + w_id, + &p.script_path, + environment, + job_id, + manifest, + manifest_key, + run_results, + run_results_key, + ) + .execute(&mut *tx) + .await?; + error::Result::Ok(Some(displaced)) + } + .await; + let displaced = match staged { + // Refused by the guard, or the write failed: nothing is committed and + // what was uploaded above has no row naming it. + Ok(None) | Err(_) => { + forget_objects(&mine).await; + return staged.map(|_| ()); + } + Ok(Some(displaced)) => displaced, + }; + // A commit that reports an error may still have committed — what was lost can + // be the acknowledgement. Dropping this run's objects would then leave the + // committed row naming objects that are gone, and every deferral would fail + // until the next publication; an orphan costs storage instead. + tx.commit().await?; + forget_objects(&displaced).await; + Ok(()) +} + +/// The environment's state, or `None` where nothing has published one. +pub(crate) async fn load( + p: &PreparedProject, + w_id: &str, + conn: &Connection, +) -> error::Result> { + let Connection::Sql(db) = conn else { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the dbt state stored for this environment, which \ + an agent worker cannot read: it reaches the database only through the API. Run this \ + script on a worker of the main group, or without `defer`" + .to_string(), + )); + }; + let environment = environment(p); + // A publication committing between the row and the objects it named drops + // those objects, so a miss is re-read rather than reported as a state that is + // not there. Re-read for as long as the row keeps MOVING: a reader takes no + // lock, so back-to-back publications can each overtake it, and a fixed one + // retry would report the second as missing. An unmoved row is the other + // answer — nothing republished, so the object really is gone. + let mut tried: Option<(Uuid, Option, Option)> = None; + for _ in 0..PUBLICATIONS_OUTRUN { + let Some(row) = sqlx::query!( + "SELECT job_id, manifest, manifest_key, run_results, run_results_key + FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(db) + .await? + else { + return Ok(None); + }; + let seen = ( + row.job_id, + row.manifest_key.clone(), + row.run_results_key.clone(), + ); + let fetched = async { + let manifest = fetch(row.manifest, row.manifest_key).await?; + let run_results = fetch(row.run_results, row.run_results_key).await?; + error::Result::Ok((manifest, run_results)) + } + .await; + match fetched { + Ok((Some(manifest), run_results)) => { + return Ok(Some(StoredState { manifest, run_results, job_id: seen.0 })) + } + Ok((None, _)) => return Ok(None), + Err(e) => { + if tried.as_ref() == Some(&seen) { + return Err(e); + } + tried = Some(seen); + } + } + } + Err(Error::internal_err( + "the dbt state for this environment was replaced faster than it could be read; run this \ + script again" + .to_string(), + )) +} + +/// How many publications a read may lose to before it gives up. Each one costs a +/// re-read, and a project publishing this often while another run defers is +/// already contending for the same relations. +const PUBLICATIONS_OUTRUN: usize = 5; + +/// A `manifest.json` for a state directory, whichever side it comes from. +/// +/// One enum because the three restores — a deferral's stored state, a retry's +/// worker-local generation, a retry's database row — differ only in where the +/// bytes are, and a second copy of the directory layout is a second chance for +/// one of them to write a directory dbt reads differently. +pub(crate) enum StateManifest { + Bytes(String), + /// A file on this worker, copied rather than read into memory: a manifest + /// grows with the project. + CopyOf(PathBuf), + None, +} + +/// Write the artifacts a dbt state directory holds, creating it if needed. +/// +/// Returns whether a `manifest.json` ended up there — a worker-local generation +/// can be pruned out from under a restore, and the caller then owes a +/// `dbt parse` for one. +pub(crate) async fn write_state_dir( + dir: &Path, + run_results: Option<&str>, + manifest: StateManifest, +) -> error::Result { + tokio::fs::create_dir_all(dir) + .await + .map_err(|e| Error::internal_err(format!("preparing the dbt state directory: {e}")))?; + if let Some(run_results) = run_results { + tokio::fs::write(dir.join("run_results.json"), run_results) + .await + .map_err(|e| Error::internal_err(format!("writing run_results.json: {e}")))?; + } + Ok(match manifest { + StateManifest::Bytes(m) => { + tokio::fs::write(dir.join("manifest.json"), m) + .await + .map_err(|e| Error::internal_err(format!("writing manifest.json: {e}")))?; + true + } + StateManifest::CopyOf(from) => tokio::fs::copy(from, dir.join("manifest.json")) + .await + .is_ok(), + StateManifest::None => false, + }) +} + +/// The advisory lock one environment's publishers take, so only one of them +/// settles the row and the objects it displaces at a time. +/// +/// Derived from the same three components as the row's key. Two environments +/// whose digests collide wait for each other, which costs a moment and nothing +/// else. +fn publication_lock(w_id: &str, script_path: &str, environment: &str) -> i64 { + let d = digest(&format!("{w_id}|{script_path}|{environment}")); + // Parsed unsigned and reinterpreted: half of all digests set the top bit, + // and read as `i64` those overflow and would collapse onto one key. + u64::from_str_radix(&d[..16], 16).unwrap_or_default() as i64 +} + +/// The object-storage key an artifact takes. +/// +/// One key per PUBLICATION, so an upload never overwrites an artifact the +/// committed row still names: a run that fails between its two uploads, or +/// between them and its row, leaves the state pointing at the pair it already +/// had. The row switches to these in one statement and the objects it displaced +/// are dropped afterwards. The path and environment are only a prefix — the row +/// is what says where an artifact is, so state that moves with a renamed script +/// keeps naming objects under the old one. Digested because a Windmill path and a +/// schema name may both carry characters an object key gives meaning to. +/// +/// The `nonce` is per EXECUTION rather than per job, because zombie recovery +/// re-runs a job under its own id: keyed on that alone, the second attempt would +/// overwrite the objects the first attempt's committed row still names, and then +/// read those same keys back as displaced and drop them. +fn object_key( + w_id: &str, + script_path: &str, + environment: &str, + job_id: &Uuid, + nonce: &Uuid, + artifact: &str, +) -> String { + format!( + "wmill_dbt_state/{w_id}/{}/{job_id}.{nonce}/{artifact}", + digest(&format!("{script_path}|{environment}")) + ) +} + +/// Put an artifact where its size says it belongs: `(inline, key)`, exactly one +/// of which is set. +#[allow(clippy::too_many_arguments)] +async fn store( + value: String, + artifact: &str, + environment: &str, + script_path: &str, + w_id: &str, + job_id: &Uuid, + nonce: &Uuid, +) -> error::Result<(Option, Option)> { + if value.len() <= *DBT_STATE_INLINE_MAX_BYTES { + return Ok((Some(value), None)); + } + let key = object_key(w_id, script_path, environment, job_id, nonce, artifact); + let size = value.len(); + if put_object(&key, value).await? { + return Ok((None, Some(key))); + } + Err(Error::BadRequest(format!( + "this project's {artifact} is {}, past the {} this instance keeps in the database, and \ + this instance has no object storage configured to hold it. Configure instance object \ + storage, or raise DBT_STATE_INLINE_MAX_BYTES", + mib(size), + mib(*DBT_STATE_INLINE_MAX_BYTES), + ))) +} + +fn mib(bytes: usize) -> String { + format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0)) +} + +/// Read an artifact back from whichever home the row names. +async fn fetch(inline: Option, key: Option) -> error::Result> { + match (inline, key) { + (Some(inline), _) => Ok(Some(inline)), + (None, Some(key)) => get_object(&key).await.map(Some), + (None, None) => Ok(None), + } +} + +/// Drop the objects nothing points at any more. Best-effort: an object left +/// behind costs storage, and there is nothing useful to do about it in the path +/// of a run that has already finished. +async fn forget_objects(keys: &[Option; 2]) { + for key in keys.iter().flatten() { + delete_object(key).await; + } +} + +/// Whether the artifact was stored. `false` means this instance has no object +/// storage to put it in. +/// +/// The INSTANCE store, where every other internal worker artifact lives — bun +/// bundles, python wheels, job logs, the global cache. Not the workspace's: +/// that bucket is the one workspace members read and write through +/// `job_helpers/*` and `wmill.write_s3_file`, so a manifest there is one any +/// member could replace, and the next deferring run would hand dbt an +/// attacker-chosen `defer_relation` for every unbuilt `ref()` while holding the +/// script's warehouse credentials. Its compiled SQL would be readable there too, +/// for a project the reader may have no access to. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn put_object(key: &str, value: String) -> error::Result { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return Ok(false); + }; + store + .put(&ObjectPath::from(key), bytes::Bytes::from(value).into()) + .await + .map_err(|e| Error::internal_err(format!("storing the dbt state at {key}: {e:#}")))?; + Ok(true) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn get_object(key: &str) -> error::Result { + let Some(store) = windmill_object_store::get_object_store().await else { + return Err(missing_storage()); + }; + let bytes = windmill_object_store::attempt_fetch_bytes(store, key).await?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::internal_err(format!("the stored dbt state is not valid UTF-8: {e}"))) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn delete_object(key: &str) { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return; + }; + if let Err(e) = store.delete(&ObjectPath::from(key)).await { + tracing::warn!("dbt: could not drop the superseded state object {key}: {e:#}"); + } +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn delete_object(_key: &str) {} + +/// A build without the instance store carries no client at all, so an oversized +/// artifact has nowhere but the row, and a row naming a key was written by a +/// worker that did have one. +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn put_object(_key: &str, _value: String) -> error::Result { + Ok(false) +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn get_object(_key: &str) -> error::Result { + Err(missing_storage()) +} + +fn missing_storage() -> Error { + Error::BadRequest( + "the dbt state for this environment is in the instance's object storage, which this \ + worker cannot reach: it is no longer configured, or this worker was built without \ + object-storage support" + .to_string(), + ) +} + +/// The stored state this run resolves its unbuilt `ref()`s through, materialised +/// into the job directory at `STATE_DIR`. +#[derive(Clone, Debug)] +pub(crate) struct Deferral { + /// The run that published the state, so the job log and the result can say + /// what this one deferred to. + pub published_by: Uuid, + /// Whether the state carries `run_results.json` beside its manifest. A build + /// recovered by node retry publishes without one, and that is the only file a + /// `result:` selector reads. + pub has_run_results: bool, +} + +/// Materialise the environment's state so `--state` has a directory to read. +/// +/// Refused rather than run without deferral where nothing is published: the run +/// would build against a `ref()` resolving into the schema it writes, and fail +/// deep inside dbt with a relation-not-found the caller has no way to connect +/// back to a missing state. +pub(crate) async fn prepare_deferral( + p: &PreparedProject, + w_id: &str, + job_dir: &str, + conn: &Connection, +) -> error::Result { + if p.script_path.is_empty() { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the state a previous run of this script \ + published, so it needs a deployed script; a preview run has no environment to have \ + published one" + .to_string(), + )); + } + // An environment is the warehouse, the target and where they RESOLVE to, and + // a `profiles.yml` that templates its schema or database resolves somewhere + // this runtime does not render. Two renderings would then share one + // environment, and a deferral after the value changed would resolve every + // unbuilt `ref()` through the previous location's manifest. + if p.templated_location { + return Err(Error::BadRequest( + "this project's profile selects its schema or database with a template, which dbt \ + renders and Windmill does not — so two environments cannot be told apart and a \ + deferral could resolve through the wrong one's manifest. Spell the target's schema \ + and database literally to use `defer`" + .to_string(), + )); + } + let Some(state) = load(p, w_id, conn).await? else { + return Err(Error::BadRequest(format!( + "no dbt state is stored for this environment ({}), so a `ref()` this run does not \ + build has no relation to resolve to. It is published by a successful run that adds \ + nothing of its own: one overriding `select` or `vars` does not publish, and neither \ + does any run of a descriptor that interpolates a `{{{{ }}}}` placeholder into `vars` \ + or a `$var:` into `env` — those describe a model set the caller's arguments decided. \ + Run this script once without `defer` and without overrides", + environment_label(p) + ))); + }; + let has_run_results = state.run_results.is_some(); + write_state_dir( + &PathBuf::from(job_dir).join(STATE_DIR), + state.run_results.as_deref(), + StateManifest::Bytes(state.manifest), + ) + .await?; + Ok(Deferral { published_by: state.job_id, has_run_results }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Every component earns its place: a deferral resolves relation NAMES, so + // state published where those names meant something else has to read as no + // state at all rather than as state that silently no longer fits. + #[test] + fn a_moved_profile_is_another_environment() { + let here = environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")); + assert_eq!( + here, + environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("other"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("dev"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("prod"), Some("marts"), Some("wh")) + ); + assert_ne!( + here, + environment_key( + Some("main"), + Some("prod"), + Some("analytics"), + Some("other_db") + ) + ); + } + + // A target name and a schema are both the user's own strings, so a component + // carrying the separator must not be able to spell another tuple's key: a + // profile moving between the two would read as the same environment and + // defer through the manifest of relations that are somewhere else. + #[test] + fn a_component_cannot_spell_another_environments_key() { + assert_ne!( + environment_key(Some("main"), Some("prod|analytics"), Some("scratch"), None), + environment_key(Some("main"), Some("prod"), Some("analytics|scratch"), None) + ); + assert_ne!( + environment_key(Some("main"), Some("prod"), Some("a"), Some("b|c")), + environment_key(Some("main"), Some("prod"), Some("a|b"), Some("c")) + ); + // A component the profile leaves out is the same environment as one it + // spells empty: there is no target named "". + assert_eq!( + environment_key(Some("main"), None, Some("a"), None), + environment_key(Some("main"), Some(""), Some("a"), Some("")) + ); + } + + // Two environments must not queue behind one advisory lock, which is what a + // digest folded through a signed parse did for every one whose top bit is + // set — half of them. + #[test] + fn each_environment_gets_its_own_publication_lock() { + let mut seen = std::collections::HashSet::new(); + for i in 0..64 { + seen.insert(publication_lock( + "ws", + "f/a/p", + &format!("main|prod|s{i}|db"), + )); + } + assert_eq!(seen.len(), 64); + assert_eq!( + publication_lock("ws", "f/a/p", "e"), + publication_lock("ws", "f/a/p", "e") + ); + } +} diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 54457f81b0..b2e69d26d6 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re // the user's own ATTACH. `// data_test` lines append verifier probes that run // against the freshly-materialized target and raise (failing the run) on // violation. Returns `None` when there is no materialize annotation or the -// target isn't a ducklake (only ducklake is materialized in v1). +// target isn't a ducklake: only ducklake has a write engine, and a `dbt://` +// target is recorded by the generic job path instead (worker.rs, +// `record_declared_warehouse_write`). fn build_materialized_query( query: &str, partition_value: Option<&str>, diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 5e9666722a..1c3bbf870f 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -9,6 +9,73 @@ use std::sync::Arc; pub const TARGET: &str = const_format::concatcp!(std::env::consts::OS, "_", std::env::consts::ARCH); +#[cfg(all(feature = "enterprise", feature = "parquet"))] +lazy_static::lazy_static! { + /// Object-store clients are built with their request timeout disabled so a large job + /// payload can stream for as long as it needs. A cache transfer must not inherit that: + /// a put or get that stalls after connecting would otherwise hold the job for its whole + /// duration limit, with nothing in the job log saying why. + pub(crate) static ref OBJECT_STORE_CACHE_IO_TIMEOUT: std::time::Duration = std::time::Duration::from_secs( + std::env::var("OBJECT_STORE_CACHE_IO_TIMEOUT_SECS") + .ok() + .and_then(|x| x.parse::().ok()) + .filter(|secs| *secs > 0) + .unwrap_or(30 * 60), + ); +} + +/// A cache transfer that did not complete: the store answered with an error, or +/// [`OBJECT_STORE_CACHE_IO_TIMEOUT`] ran out first. Nothing is logged on the way out: a failed +/// download is usually an ordinary miss, so each call site decides which outcome gets a line, +/// and logs it once. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +pub(crate) enum CacheIoError { + TimedOut { what: String, path: String, secs: u64 }, + Failed(error::Error), +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +impl std::fmt::Display for CacheIoError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CacheIoError::TimedOut { what, path, secs } => write!( + f, + "{what} {path} in the object store cache timed out after {secs}s \ + (OBJECT_STORE_CACHE_IO_TIMEOUT_SECS)" + ), + CacheIoError::Failed(e) => write!(f, "{e:#}"), + } + } +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +impl From for error::Error { + fn from(e: CacheIoError) -> Self { + match e { + CacheIoError::Failed(e) => e, + timed_out => error::Error::ExecutionErr(timed_out.to_string()), + } + } +} + +/// Runs one object-store cache transfer under [`OBJECT_STORE_CACHE_IO_TIMEOUT`]. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +pub(crate) async fn bounded_cache_io( + what: &str, + path: &str, + io: impl std::future::Future>, +) -> Result { + let timeout = *OBJECT_STORE_CACHE_IO_TIMEOUT; + match tokio::time::timeout(timeout, io).await { + Ok(result) => result.map_err(CacheIoError::Failed), + Err(_) => Err(CacheIoError::TimedOut { + what: what.to_string(), + path: path.to_string(), + secs: timeout.as_secs(), + }), + } +} + #[cfg(all(feature = "enterprise", feature = "parquet"))] pub async fn build_tar_and_push( s3_client: Arc, @@ -56,17 +123,20 @@ pub async fn build_tar_and_push( // let s3_client = s3_settings.as_ref().ok_or_else(|| { // error::Error::ExecutionErr("Failed to read s3 cache settings".to_string()) // })?; - if let Err(e) = s3_client - .put( - &Path::from(format!( - "/tar/{}/{lang}/{folder_name}.tar", - if platform_agnostic { "" } else { TARGET } - )), - std::fs::read(&tar_path)?.into(), - ) - .await - { - tracing::info!("Failed to put tar to s3: {tar_path}. Error: {:?}", e); + let remote_path = format!( + "/tar/{}/{lang}/{folder_name}.tar", + if platform_agnostic { "" } else { TARGET } + ); + let tar_bytes = std::fs::read(&tar_path)?; + let put = bounded_cache_io("uploading", &remote_path, async { + s3_client + .put(&Path::from(remote_path.as_str()), tar_bytes.into()) + .await + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e) = put { + tracing::info!("Failed to put tar to s3: {tar_path}. Error: {e}"); return Err(error::Error::ExecutionErr(format!( "Failed to put tar to s3: {tar_path}" ))); @@ -109,7 +179,12 @@ pub async fn pull_from_tar( "tar/{}/{lang}/{folder_name}.tar", if platform_agnostic { "" } else { TARGET } ); - let bytes = attempt_fetch_bytes(client, &tar_path).await?; + let bytes = bounded_cache_io( + "downloading", + &tar_path, + attempt_fetch_bytes(client, &tar_path), + ) + .await?; extract_tar(bytes, &folder).map_err(|e| { tracing::error!("Failed to extract piptar {folder_name}. Error: {:?}", e); @@ -147,17 +222,29 @@ pub fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { Ok(()) } -/// Two-tier cache load: check local disk first, then fall back to instance object store. +/// Two-tier cache load: check local disk first, then fall back to the shared object store. +/// +/// "Shared" is the worker group's own store when its config overrides one, the instance store +/// otherwise — see [`windmill_object_store::get_cache_object_store`]. /// Returns `(hit, log_message)`. pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let started = std::time::Instant::now(); - if let Ok(mut x) = windmill_object_store::attempt_fetch_bytes(os, _remote_path).await { + let fetched = bounded_cache_io( + "downloading", + _remote_path, + windmill_object_store::attempt_fetch_bytes(os, _remote_path), + ) + .await; + if let Err(e @ CacheIoError::TimedOut { .. }) = &fetched { + tracing::error!("{e}"); + } + if let Ok(mut x) = fetched { if is_dir { // Extract into a sibling temp dir then atomically publish it, // so a concurrent cold-load gating on metadata(bin_path) never @@ -200,13 +287,15 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bo } } -/// Whether this worker can push to the instance object store at all — the features are +/// Whether this worker can push to the shared object store at all — the features are /// compiled in and a store is loaded. False on builds without them, where `save_cache` /// only ever writes to the worker's own disk. pub async fn object_store_available() -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] { - windmill_object_store::get_object_store().await.is_some() + windmill_object_store::get_cache_object_store() + .await + .is_some() } #[cfg(not(all(feature = "enterprise", feature = "parquet")))] { @@ -214,20 +303,26 @@ pub async fn object_store_available() -> bool { } } -/// Whether a binary/bundle is in the instance object store, ignoring the local cache. +/// Whether a binary/bundle is in the shared object store, ignoring the local cache. /// /// The deploy-time prebuild asks this rather than [`exists_in_cache`]: a copy on the /// building worker's own disk is exactly the state the prebuild exists to fix, so /// answering from it would latch a failed upload into a permanent skip. pub async fn exists_in_object_store(_remote_path: &str) -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { - return os - .head(&windmill_object_store::object_store_reexports::Path::from( + if let Some(os) = windmill_object_store::get_cache_object_store().await { + let head = bounded_cache_io("checking", _remote_path, async { + os.head(&windmill_object_store::object_store_reexports::Path::from( _remote_path, )) .await - .is_ok(); + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e @ CacheIoError::TimedOut { .. }) = &head { + tracing::error!("{e}"); + } + return head.is_ok(); } false } @@ -241,30 +336,36 @@ pub async fn ensure_pushed_to_object_store(remote_path: &str) -> error::Result<( return Ok(()); } Err(error::Error::ExecutionErr(format!( - "the binary was built but did not reach the instance object store at {remote_path}, \ + "the binary was built but did not reach the object store at {remote_path}, \ so no other worker can load it" ))) } -/// Check whether a binary/bundle exists in local cache or instance object store. +/// Check whether a binary/bundle exists in local cache or the shared object store. pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { if tokio::fs::metadata(&bin_path).await.is_ok() { return true; } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { - return os - .get(&windmill_object_store::object_store_reexports::Path::from( + if let Some(os) = windmill_object_store::get_cache_object_store().await { + let get = bounded_cache_io("checking", _remote_path, async { + os.get(&windmill_object_store::object_store_reexports::Path::from( _remote_path, )) .await - .is_ok(); + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e @ CacheIoError::TimedOut { .. }) = &get { + tracing::error!("{e}"); + } + return get.is_ok(); } return false; } } -/// Two-tier cache write: upload to instance object store, then copy to local disk. +/// Two-tier cache write: upload to the shared object store, then copy to local disk. pub async fn save_cache( local_cache_path: &str, _remote_cache_path: &str, @@ -275,7 +376,7 @@ pub async fn save_cache( let mut _cached_to_s3 = false; #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { use windmill_object_store::object_store_reexports::Path; let file_to_cache = if is_dir { let tar_path = format!( @@ -302,17 +403,15 @@ pub async fn save_cache( origin.to_owned() }; - if let Err(e) = os - .put( - &Path::from(_remote_cache_path), - std::fs::read(&file_to_cache)?.into(), - ) - .await - { - tracing::error!( - "Failed to put bin to object store: {_remote_cache_path}. Error: {:?}", - e - ); + let bytes = std::fs::read(&file_to_cache)?; + let put = bounded_cache_io("uploading", _remote_cache_path, async { + os.put(&Path::from(_remote_cache_path), bytes.into()) + .await + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e) = put { + tracing::error!("Failed to put bin to object store: {_remote_cache_path}. Error: {e}"); } else { _cached_to_s3 = true; if is_dir { diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index e3e7868548..6059936af6 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -33,9 +33,11 @@ pub mod common; mod config; mod csharp_executor; +mod dbt_column_index; mod dbt_engine; mod dbt_executor; mod dbt_profiles; +mod dbt_state; #[cfg(feature = "private")] mod dedicated_worker_ee; mod dedicated_worker_oss; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 1ede0933cb..03328b82cb 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -14,7 +14,7 @@ use windmill_common::{ }; use windmill_queue::MiniPulledJob; -use windmill_parser::Typ; +use windmill_parser::{MainArgSignature, Typ}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -40,6 +40,43 @@ lazy_static::lazy_static! { const COMPOSER_LOCK_SPLIT: &str = "\nLOCK\n"; +static PHP_PARSER_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + +pub(crate) async fn parse_php_signature( + code: &str, + main_override: Option, +) -> Result { + parse_php_signature_with_slot(code, main_override, &PHP_PARSER_SLOT).await +} + +async fn parse_php_signature_with_slot( + code: &str, + main_override: Option, + slot: &'static tokio::sync::Semaphore, +) -> Result { + let acquire = slot.acquire(); + tokio::pin!(acquire); + // Retain the acquisition across the warning to preserve its FIFO queue position. + let permit = tokio::select! { + permit = &mut acquire => permit, + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => { + tracing::warn!("Waiting over a second for PHP signature parser capacity"); + acquire.await + } + } + .map_err(to_anyhow)?; + let code = code.to_owned(); + tokio::task::spawn_blocking(move || { + // Parsing walks the entire AST. Keep its stack off async workers and retain + // the process-wide CPU limit even if the awaiting job is cancelled. + let _permit = permit; + windmill_parser_php::parse_php_signature(&code, main_override) + }) + .await + .map_err(|e| error::Error::internal_err(format!("PHP signature parsing task failed: {e}")))? + .map_err(Into::into) +} + pub fn parse_php_imports(code: &str) -> anyhow::Result> { let find_requirements = code .lines() @@ -333,11 +370,9 @@ pub async fn handle_php_job( let main_override = job.script_entrypoint_override.as_deref(); let write_wrapper_f = async { - let args = windmill_parser_php::parse_php_signature( - inner_content, - main_override.map(ToString::to_string), - )? - .args; + let args = parse_php_signature(inner_content, main_override.map(ToString::to_string)) + .await? + .args; let args_to_include = args .iter() @@ -491,3 +526,51 @@ try {{ .await?; read_result(job_dir, None).await } + +#[cfg(test)] +mod tests { + use super::parse_php_signature_with_slot; + + #[test] + fn cancelled_parse_retains_slot_until_blocking_work_finishes() { + static SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + let runtime = tokio::runtime::Builder::new_current_thread() + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let blocker = runtime.spawn_blocking(move || { + started_tx.send(()).unwrap(); + let _ = release_rx.recv(); + }); + + runtime.block_on(async { + started_rx.await.unwrap(); + let parse = tokio::spawn(parse_php_signature_with_slot( + ") { use crate::global_cache::build_tar_and_push; - use windmill_object_store::get_object_store; + use windmill_object_store::get_cache_object_store; while let Some(task) = rx.recv().await { - if let Some(os) = get_object_store().await { + if let Some(os) = get_cache_object_store().await { match build_tar_and_push(os, task.venv_path.clone(), task.cache_dir, None, false).await { Ok(()) => { @@ -144,7 +145,7 @@ pub fn has_relative_imports(content: &str) -> bool { use crate::global_cache::pull_from_tar; #[cfg(all(feature = "enterprise", feature = "parquet"))] -use windmill_object_store::OBJECT_STORE_SETTINGS; +use windmill_object_store::get_cache_object_store; use crate::{ common::{ @@ -227,9 +228,9 @@ fn filter_pip_local_dependencies(lines: Vec) -> (Vec, Vec, compiled_deps: &[Regex]) -> (Vec, Vec) { - let (ignored, kept): (Vec, Vec) = lines - .into_iter() - .partition(|s| !s.starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s))); + let (ignored, kept): (Vec, Vec) = lines.into_iter().partition(|s| { + !s.trim_start().starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s)) + }); (kept, ignored) } @@ -323,6 +324,11 @@ pub async fn uv_pip_compile( "compile", "-q", "--no-header", + // The `#`-line filter applied to the output below only catches whole-line + // annotations, and uv's annotation style is configurable: `[pip] + // annotation-style = "line"` in the worker HOME's uv.toml emits them inline + // ("anyio==4.15.1 # via httpx"), which that filter keeps. + "--no-annotate", file, "--strip-extras", "-o", @@ -2332,6 +2338,33 @@ async fn spawn_uv_install( } } +/// First field (the path) of a wheel RECORD line. RECORD is CSV (PEP 376 / +/// RFC 4180): a path containing a comma or a double quote is written quoted, +/// with inner quotes doubled, so splitting on the first comma turns such an +/// entry into a name that never exists on disk. +fn record_first_field(line: &str) -> Option { + let Some(quoted) = line.strip_prefix('"') else { + return line + .split(',') + .next() + .filter(|p| !p.is_empty()) + .map(str::to_owned); + }; + let mut field = String::new(); + let mut chars = quoted.chars(); + while let Some(c) = chars.next() { + if c != '"' { + field.push(c); + } else if chars.as_str().starts_with('"') { + chars.next(); + field.push('"'); + } else { + return Some(field).filter(|f| !f.is_empty()); + } + } + None +} + /// Verify that every file listed in the wheel's RECORD exists on disk under /// `venv_p`. Used as a structural integrity check after both a successful /// `pull_from_tar` (object-store cache hit) and a successful local @@ -2380,9 +2413,9 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { if trimmed.is_empty() { continue; } - let rel_path = match trimmed.split(',').next() { - Some(p) if !p.is_empty() => p, - _ => continue, + let rel_path = match record_first_field(trimmed) { + Some(p) => p, + None => continue, }; // Defensive: skip absolute paths or escaping entries — we only // validate package-relative files. @@ -2391,7 +2424,7 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { } let full = format!("{venv_p}/{rel_path}"); if tokio::fs::metadata(&full).await.is_err() { - missing.push(rel_path.to_string()); + missing.push(rel_path); // Bound error size in pathological cases (e.g. wholly empty dir). if missing.len() >= 10 { missing.push("...".to_string()); @@ -2449,7 +2482,7 @@ pub async fn handle_python_reqs( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if OBJECT_STORE_SETTINGS.read().await.is_none() { + if get_cache_object_store().await.is_none() { (s3_pull, s3_push) = (false, false); } @@ -2520,11 +2553,23 @@ pub async fn handle_python_reqs( // Find out if there is already cached dependencies // If so, skip them let mut in_cache = vec![]; + if requirements + .iter() + .any(|r| lockfile_line_has_continuation(r)) + { + tracing::warn!(workspace_id = %w_id, job_id = %job_id, "lockfile continues entries across lines; the continued lines are dropped"); + append_logs( + job_id, + w_id, + "\n[!] lockfile continues entries across lines and the continued lines are dropped: `--hash=` pins, extras and markers written that way do not apply\n".to_string(), + conn, + ) + .await; + } for req in &requirements { - // Ignore python version annotation backed into lockfile - if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { + let Some(req) = requirement_from_lockfile_line(req) else { continue; - } + }; let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( @@ -2907,7 +2952,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet"))] if is_not_pro { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(Error::from(anyhow::anyhow!("S3 pull was canceled"))), @@ -3734,6 +3779,44 @@ mod tests { .is_ok()); } + #[tokio::test] + async fn test_verify_wheel_record_accepts_csv_quoted_path() { + let dir = tempfile::tempdir().unwrap(); + // A path containing a comma is CSV-quoted in RECORD (wcwidth 0.8.3 + // ships `wcwidth/textwrap.py,cover`). Splitting on the first comma + // looked for `"pkg/textwrap.py` and rejected a complete install. + write_fake_wheel( + dir.path(), + &["pkg/textwrap.py", "pkg/textwrap.py,cover"], + &[ + "pkg/textwrap.py,sha256=aaa,1", + "\"pkg/textwrap.py,cover\",sha256=bbb,1", + "pkg-1.0.0.dist-info/RECORD,,", + ], + ); + assert!(verify_wheel_record(dir.path().to_str().unwrap()) + .await + .is_ok()); + } + + #[test] + fn test_record_first_field_unquotes_rfc4180() { + assert_eq!( + record_first_field("pkg/a.py,sha256=x,1").as_deref(), + Some("pkg/a.py") + ); + assert_eq!( + record_first_field("\"pkg/a.py,cover\",sha256=x,1").as_deref(), + Some("pkg/a.py,cover") + ); + assert_eq!( + record_first_field("\"pkg/say \"\"hi\"\".py\",sha256=x,1").as_deref(), + Some("pkg/say \"hi\".py") + ); + assert_eq!(record_first_field(",,"), None); + assert_eq!(record_first_field("\"unterminated,sha256=x,1"), None); + } + // Regression tests for the concurrent-install guard. Two jobs installing the // same uncached dep into the shared `venv_p` used to race uv's `--reinstall`, // corrupting the on-disk wheel and failing with "Env installation did not diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 647d569cbe..1529d474fa 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -16,10 +16,6 @@ use windmill_common::otel_oss::FutureExt; use uuid::Uuid; -/// Set by the result processor when a WAC child completion makes suspend reach 0, -/// signaling the worker main loop to check for suspended jobs immediately. -pub static WAC_SUSPEND_READY: AtomicBool = AtomicBool::new(false); - use windmill_common::{ add_time, error::{self, Error}, @@ -36,8 +32,8 @@ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, - CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, - WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, + parse_result_object, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, + ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -76,13 +72,11 @@ struct NestedErrorMessage { /// named `name`/`message`), and we want OTel to record the ManualFailure /// rather than the user's sibling fields. fn extract_error_message(raw: &str) -> Option { - let nested = serde_json::from_str::(raw) - .ok() - .map(|n| n.error); + let nested = parse_result_object::(raw).map(|n| n.error); if matches!(&nested, Some(em) if em.name == MANUAL_FAILURE_ERROR_NAME) { return nested; } - if let Ok(em) = serde_json::from_str::(raw) { + if let Some(em) = parse_result_object::(raw) { return Some(em); } nested @@ -816,8 +810,19 @@ pub async fn handle_receive_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] #[derive(serde::Deserialize)] struct GitSyncCheck { - check_run_id: i64, - repo_url: String, + /// Absent when the repository's host has no check surface (GitLab): the + /// result then reaches the pull request through the managed comment alone. + #[serde(default)] + check_run_id: Option, + /// Only markers written before the repository URL moved out of job args + /// carry one; the resource path on the job is what is used now. + #[serde(default)] + repo_url: Option, + /// Host and path of the repository the check was created on, with no + /// credential in it. The resource path is mutable, so this is what proves + /// the resource still points where the check lives. + #[serde(default)] + repo: Option, #[serde(default)] pr_number: Option, #[serde(default)] @@ -972,19 +977,21 @@ mod git_sync_check_tests { } } -/// When an auto-pull job (carrying `__git_sync_auto_pull`) fails, roll the +/// When an auto-pull job (carrying `__git_sync_auto_pull`) completes: on success, +/// record the commit as a head the workspace reflects; on failure, roll the /// optimistic `last_synced_sha` advance back to the pre-pull value so the commit /// is retried instead of being silently treated as synced, and record the failure. +/// The recorded commit is the one the pull script reports having checked out +/// (`{sha, branch}` in its result): the branch can move between the observation +/// the marker holds and the clone. A result without it falls back to the marker. #[cfg(all(feature = "enterprise", feature = "private"))] async fn maybe_reconcile_git_sync_auto_pull( db: &DB, job_id: &uuid::Uuid, workspace_id: &str, success: bool, + result: &str, ) { - if success { - return; // the optimistic synced state is already correct - } let marker: Option = match sqlx::query_scalar!( "SELECT args->'__git_sync_auto_pull' FROM v2_job WHERE id = $1", job_id @@ -1004,12 +1011,50 @@ async fn maybe_reconcile_git_sync_auto_pull( #[derive(serde::Deserialize)] struct AutoPullMarker { repo_resource_path: String, + branch: Option, + head_sha: Option, #[serde(default)] prev_synced: std::collections::HashMap, } let Ok(m) = serde_json::from_value::(marker) else { return; }; + if success { + // The optimistic synced state is already correct; record that the workspace + // now reflects the commit, which the PR CI-test check waits for. + #[derive(serde::Deserialize)] + struct PullResult { + sha: Option, + branch: Option, + } + let applied = serde_json::from_str::(result).ok(); + let branch = applied + .as_ref() + .and_then(|r| r.branch.as_deref()) + .or(m.branch.as_deref()); + let sha = applied + .as_ref() + .and_then(|r| r.sha.as_deref()) + .or(m.head_sha.as_deref()); + if let (Some(branch), Some(sha)) = (branch, sha) { + if let Err(e) = windmill_git_sync::record_synced_head( + db, + workspace_id, + &m.repo_resource_path, + branch, + sha, + "pull", + Some(*job_id), + ) + .await + { + tracing::warn!( + "git auto-pull: failed to record synced head {sha} on {branch}: {e:#}" + ); + } + } + return; + } windmill_git_sync::record_auto_pull_failure( db, workspace_id, @@ -1098,6 +1143,70 @@ fn git_sync_push_result_pushed(result: &str) -> Option { .as_bool() } +/// When a git-sync push job pushed a commit, record it as a head the workspace +/// reflects, the way a successful pull records the commit it applied. The PR +/// CI-test check waits for that record. Best-effort: failures are logged, never +/// propagated. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maybe_record_git_sync_pushed_head( + db: &DB, + job_id: &uuid::Uuid, + workspace_id: &str, + result: &str, +) { + #[derive(serde::Deserialize)] + struct PushResult { + pushed: bool, + sha: Option, + branch: Option, + #[serde(default)] + rebased: bool, + } + let Ok(PushResult { pushed: true, sha: Some(sha), branch: Some(branch), rebased }) = + serde_json::from_str::(result) + else { + return; + }; + // A push that had to rebase sits on commits this workspace has not pulled, so the + // pushed head is not something it reflects yet; the pull those commits trigger + // records the head once they are in. + if rebased { + tracing::info!( + "git sync push: {sha} on {branch} was rebased onto unpulled commits; not recording it as synced for {workspace_id}" + ); + return; + } + let repo_path = match sqlx::query_scalar!( + "SELECT args->>'repo_url_resource_path' FROM v2_job WHERE id = $1", + job_id + ) + .fetch_optional(db) + .await + { + Ok(Some(Some(p))) => p, + Ok(_) => return, + Err(e) => { + tracing::error!("git sync push: failed to read job args: {e:#}"); + return; + } + }; + if let Err(e) = windmill_git_sync::record_synced_head( + db, + workspace_id, + &repo_path, + &branch, + &sha, + "push", + Some(*job_id), + ) + .await + { + tracing::warn!( + "git sync push: failed to record pushed head {sha} on {branch} for {workspace_id}/{repo_path}: {e:#}" + ); + } +} + /// When a git-sync push job carrying `__git_sync_open_pr` succeeds, open (or /// reopen) the PR for the branch it pushed: `wm-fork//` for a fork /// deploy, `wm_deploy/**` for a promotion deploy. Runs outbound with the @@ -1162,18 +1271,18 @@ async fn maybe_open_git_sync_deploy_pr( }; // Base = the tracked branch (resource branch, else the repo default). Also - // acts as the app-backed gate: PR creation needs the installation token. - let base = match windmill_common::git_sync_ee::get_app_repo_head_for_autopull( + // acts as the gate: PR creation needs a credential the server itself holds. + let base = match windmill_common::git_sync_ee::managed_pr_base_branch( db, workspace_id, &repo_path, ) .await { - Ok(Some((branch, _))) => branch, + Ok(Some(branch)) => branch, Ok(None) => { tracing::warn!( - "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but is not GitHub-App-backed; skipping (connect the repo through the GitHub App, or use the open-pr-on-commit workflow)" + "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but the server holds no credential for it; skipping (connect the repo through the GitHub App or a GitLab token, or use the open-pr-on-commit workflow)" ); return; } @@ -1373,22 +1482,59 @@ async fn maybe_post_git_sync_check( (None, Some(deploy)) => (true, deploy), (None, None) => return, }; - let Ok(mut check) = serde_json::from_value::(marker) else { + let Ok(check) = serde_json::from_value::(marker) else { return; }; - // Markers carry the literal resource URL (job args are persisted, so a - // `$var:`-resolved URL must not land there); interpolate before calling - // GitHub. - check.repo_url = - match windmill_common::variables::get_variable_or_self(check.repo_url, db, workspace_id) - .await - { - Ok(u) => u, - Err(e) => { - tracing::error!("git sync-check: cannot interpolate repo url: {e:#}"); - return; - } - }; + // Job args are persisted, so the repository URL is not among them: it is + // re-resolved here from the resource path the pull job carries. A marker + // written before that change still has the URL, and is honoured until the + // last such job has drained. + // The resource path is mutable, so it is only trusted when the marker also + // carries the identity to check it against. A marker written before that + // identity existed keeps using the URL it captured at enqueue, which cannot + // have been repointed since. + let repo_url = match ( + check.repo.is_some(), + row.repo_path.as_deref(), + check.repo_url.clone(), + ) { + // The resource path is mutable, so following it is only safe when the + // marker also carries the identity to check the result against. + (true, Some(path), _) => { + windmill_common::git_sync_ee::resolve_repo_url_interpolated(db, workspace_id, path) + .await + } + // A marker written before that identity existed captured the URL itself, + // which cannot have been repointed since. + (_, _, Some(url)) => { + windmill_common::variables::get_variable_or_self(url, db, workspace_id).await + } + // Neither: nothing here can prove which repository this check belongs to, + // and resolving the path anyway is how a preview reaches the wrong one. + // Leaving the check unfinished is the safe failure. + _ => { + tracing::error!( + "git sync-check: the marker carries neither a repository identity nor a url; not acting on it" + ); + return; + } + }; + let repo_url = match repo_url { + Ok(u) => u, + Err(e) => { + tracing::error!("git sync-check: cannot resolve repo url: {e:#}"); + return; + } + }; + // A resource repointed while the diff was running would otherwise close a + // check, or post a preview, on a repository that has nothing to do with it. + if check.repo.is_some() && windmill_common::git_sync_ee::repo_identity(&repo_url) != check.repo + { + tracing::warn!( + "git sync-check: the repository moved since the check was created; leaving it alone" + ); + return; + } // "In sync" on a PR that visibly changes files reads as a bug when those // files are outside the repo's sync filters — say what the scope is. let scope_note = if !is_deploy && success { @@ -1451,7 +1597,7 @@ async fn maybe_post_git_sync_check( ( "failure", "Merge conflicts with the base branch".to_string(), - "This PR cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." + "This branch cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." .to_string(), ) } else if pr_check_error.as_deref() == Some("PR_HEAD_REF_UNAVAILABLE") { @@ -1461,7 +1607,7 @@ async fn maybe_post_git_sync_check( ( "neutral", "Could not compute the deploy diff".to_string(), - "Windmill could not fetch this PR's head or enough history from GitHub to compute its merge with the base. Push again to re-run this check." + "Windmill could not fetch this branch's head, or enough history, to compute its merge with the base. Push again to re-run this check." .to_string(), ) } else if pr_check_error.is_some() { @@ -1484,20 +1630,20 @@ async fn maybe_post_git_sync_check( "success", "In sync".to_string(), format!( - "Merging this PR would make no changes to the workspace.{}", + "Merging this branch would make no changes to the workspace.{}", scope_note.as_deref().unwrap_or_default() ), ), Some((changes, settings_changed)) => { let mut lines = vec![format!( - "Merging this PR would apply {} change(s) to the workspace:\n", + "Merging this branch would apply {} change(s) to the workspace:\n", changes.len() )]; lines.extend(format_change_list(&changes)); if settings_changed { lines.push(match check.wmill_yaml_changed { - Some(true) => "\nThis PR changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), - Some(false) => "\nIndependent of this PR, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), + Some(true) => "\nThis branch changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), + Some(false) => "\nIndependent of this branch, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), None => "\nA pull also updates the workspace's git-sync settings to match the repo's wmill.yaml.".to_string(), }); } @@ -1520,19 +1666,21 @@ async fn maybe_post_git_sync_check( Some(url) => format!("{summary}\n\n[See the job in Windmill]({url})"), None => summary.clone(), }; - if let Err(e) = windmill_common::git_sync_ee::update_check_run( - db, - workspace_id, - &check.repo_url, - check.check_run_id, - conclusion, - &title, - &check_summary, - job_url.as_deref(), - ) - .await - { - tracing::error!("git sync-check: failed to update check run: {e:#}"); + if let Some(check_run_id) = check.check_run_id { + if let Err(e) = windmill_common::git_sync_ee::update_check_run( + db, + workspace_id, + &repo_url, + check_run_id, + conclusion, + &title, + &check_summary, + job_url.as_deref(), + ) + .await + { + tracing::error!("git sync-check: failed to update check run: {e:#}"); + } } // Phase 4 also maintains ONE managed comment on the PR (Cloudflare @@ -1556,7 +1704,7 @@ async fn maybe_post_git_sync_check( if let Err(e) = windmill_common::git_sync_ee::upsert_pr_comment( db, workspace_id, - &check.repo_url, + &repo_url, pr_number, marker, &body, @@ -1640,7 +1788,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration, wac_job_ids) = add_completed_job( + let (_, duration) = add_completed_job( db, &job, true, @@ -1657,8 +1805,26 @@ pub async fn process_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { maybe_post_git_sync_check(db, &job_id, &workspace_id, true, result.get()).await; + maybe_reconcile_git_sync_auto_pull(db, &job_id, &workspace_id, true, result.get()) + .await; + maybe_record_git_sync_pushed_head(db, &job_id, &workspace_id, result.get()).await; maybe_open_git_sync_deploy_pr(db, &job_id, &workspace_id, result.get()).await; } + // A CI test job just finished: advance any open "Windmill CI tests" PR check for + // its workspace. Detached, since concluding a check calls GitHub and this loop + // completes jobs serially; the evaluation is idempotent and the poller retries. + #[cfg(all(feature = "enterprise", feature = "private"))] + if job + .trigger_kind + .as_ref() + .is_some_and(|k| k.is(windmill_common::jobs::JobTriggerKind::CiTest)) + { + let db = db.clone(); + let w_id = workspace_id.clone(); + tokio::spawn(async move { + windmill_git_sync::evaluate_and_conclude_ci_test_checks(&db, &w_id).await + }); + } // Asset-trigger fan-out: best-effort, never propagates errors. // Internal eligibility checks gate to top-level Script/Preview runs; @@ -1703,29 +1869,6 @@ pub async fn process_completed_job( } return Ok(r); } - } else if let Some(parent_job) = parent_job { - // wac_job_ids is piggybacked from the duration write in - // add_completed_job — no extra query needed. - if let Some(job_ids) = wac_job_ids { - if let Ok(Some(_)) = handle_wac_child_completion( - db, - &job_id, - parent_job, - &workspace_id, - result, - true, - job_ids, - ) - .await - { - if let Some(done_tx) = done_tx { - done_tx - .send(()) - .expect("done receiver should still be alive"); - } - return Ok(None); - } - } } } else { // The result already carries our injected @@ -1769,7 +1912,20 @@ pub async fn process_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { maybe_post_git_sync_check(db, &job.id, &job.workspace_id, false, result.get()).await; - maybe_reconcile_git_sync_auto_pull(db, &job.id, &job.workspace_id, false).await; + maybe_reconcile_git_sync_auto_pull(db, &job.id, &job.workspace_id, false, "").await; + } + // A failed CI test job also settles its check; same detached advance as on success. + #[cfg(all(feature = "enterprise", feature = "private"))] + if job + .trigger_kind + .as_ref() + .is_some_and(|k| k.is(windmill_common::jobs::JobTriggerKind::CiTest)) + { + let db = db.clone(); + let w_id = job.workspace_id.clone(); + tokio::spawn(async move { + windmill_git_sync::evaluate_and_conclude_ci_test_checks(&db, &w_id).await + }); } if job.is_flow_step() { if let Some(parent_job) = job.parent_job { @@ -1809,227 +1965,11 @@ pub async fn process_completed_job( } return Ok(r); } - } else if let Some(parent_job) = job.parent_job { - // WAC child failed — query job_ids from parent (errors are rare, - // so the extra read is acceptable here). - let job_ids_json: Option> = sqlx::query_scalar( - "SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ - FROM v2_job_status WHERE id = $1", - ) - .bind(&parent_job) - .fetch_optional(db) - .await?; - if let Some(Some(job_ids)) = job_ids_json { - if let Ok(Some(_)) = handle_wac_child_completion( - db, - &job.id, - parent_job, - &job.workspace_id, - downstream_result, - false, - job_ids, - ) - .await - { - if let Some(done_tx) = done_tx { - done_tx - .send(()) - .expect("done receiver should still be alive"); - } - return Ok(None); - } - } } } return Ok(None); } -/// Handle a WAC v2 child job completion. -/// Returns Ok(Some(())) if the parent was a WAC job and was handled, -/// Ok(None) if the parent is not a WAC job (caller should fall through). -/// -/// CONCURRENCY: Multiple parallel children may complete simultaneously on -/// different workers. We use atomic SQL operations throughout: -/// - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))` -/// — PostgreSQL serialises concurrent UPDATEs on the same row, so each -/// worker sees the previous worker's writes. -/// - The suspend counter (set to N at dispatch time) is decremented atomically -/// with `RETURNING` to determine the "all done" condition. -pub(crate) async fn handle_wac_child_completion( - db: &DB, - child_job_id: &Uuid, - parent_job_id: Uuid, - workspace_id: &str, - result: Arc>, - success: bool, - job_ids_value: Value, -) -> error::Result> { - let job_ids = match job_ids_value { - Value::Object(m) => m, - _ => return Ok(None), // Not a WAC parent or no pending steps - }; - - let child_id_str = child_job_id.to_string(); - let step_key = job_ids.iter().find_map(|(key, val)| { - if val.as_str() == Some(&child_id_str) { - Some(key.clone()) - } else { - None - } - }); - - let step_key = match step_key { - Some(k) => k, - None => { - if !success { - // No step key and failed — can't store error, fail parent immediately - tracing::error!( - parent_job = %parent_job_id, - child_job = %child_job_id, - "WAC v2 child job failed but no step key found, failing parent" - ); - sqlx::query!( - "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", - parent_job_id, - ) - .execute(db) - .await?; - let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?; - if let Some(parent_mini) = parent_mini { - let child_err: Value = - serde_json::from_str(result.get()).unwrap_or(Value::Null); - let err_value = json!({ - "message": format!("WAC child job {} failed (no step key)", child_job_id), - "error": child_err, - }); - let _ = windmill_queue::add_completed_job_error( - db, - &parent_mini, - 0, - None, - err_value, - "wac_child_handler", - false, - None, - ) - .await; - } - return Ok(Some(())); - } - tracing::warn!( - parent_job = %parent_job_id, - child_job = %child_job_id, - "WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang" - ); - // Still decrement suspend so the parent doesn't hang indefinitely - let _ = sqlx::query_scalar!( - "UPDATE v2_job_queue \ - SET suspend = GREATEST(suspend - 1, 0) \ - WHERE id = $1 \ - RETURNING suspend", - parent_job_id, - ) - .fetch_optional(db) - .await?; - return Ok(Some(())); - } - }; - - // Build result — wrap errors with _error marker so workflow try/catch can handle them - let result_value: Value = if success { - serde_json::from_str(result.get()).unwrap_or(Value::Null) - } else { - let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null); - tracing::info!( - parent_job = %parent_job_id, - child_job = %child_job_id, - step_key = %step_key, - "WAC v2 child job failed, storing error for workflow try/catch" - ); - windmill_common::wac::wac_failure_record( - &step_key, - Some(&child_job_id.to_string()), - &child_err, - ) - }; - - tracing::info!( - parent_job = %parent_job_id, - child_job = %child_job_id, - step_key = %step_key, - success = success, - "WAC v2 child job completed" - ); - - // Use a transaction to ensure completed_steps merge + suspend decrement - // are atomic. Without this, a crash between the two could strand the parent. - let result_json = serde_json::to_value(&result_value) - .map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?; - - let mut tx = db.begin().await?; - - // Merge the completed step into the checkpoint. - // Uses `|| jsonb_build_object(key, value)` so concurrent children on - // different workers don't overwrite each other — PostgreSQL serialises - // concurrent UPDATEs on the same row and each sees the previous write. - sqlx::query( - "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( - workflow_as_code_status, - '{_checkpoint,completed_steps}', - COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) - || jsonb_build_object($2::text, $3::jsonb) - ) WHERE id = $1", - ) - .bind(&parent_job_id) - .bind(&step_key) - .bind(&result_json) - .execute(&mut *tx) - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?; - - // Decrement the suspend counter. The counter was set to N (number of - // children) at dispatch time. When it reaches 0 all children are done. - // Keep suspend_until non-null so the suspended pull query - // (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent. - let new_suspend: Option = sqlx::query_scalar!( - "UPDATE v2_job_queue \ - SET suspend = GREATEST(suspend - 1, 0) \ - WHERE id = $1 \ - RETURNING suspend", - parent_job_id, - ) - .fetch_optional(&mut *tx) - .await?; - - let all_done = new_suspend == Some(0); - - if all_done { - // Clear pending_steps from checkpoint since all children are complete. - // This is cosmetic — the next replay will overwrite it anyway — but - // keeps the checkpoint clean for frontend display. - let _ = sqlx::query( - "UPDATE v2_job_status SET workflow_as_code_status = \ - workflow_as_code_status #- '{_checkpoint,pending_steps}' \ - WHERE id = $1", - ) - .bind(&parent_job_id) - .execute(&mut *tx) - .await; - } - - tx.commit().await?; - - if all_done { - tracing::info!( - parent_job = %parent_job_id, - "WAC v2 all child jobs completed, unsuspending parent" - ); - WAC_SUSPEND_READY.store(true, Ordering::Relaxed); - } - - Ok(Some(())) -} - pub async fn handle_non_flow_job_error( db: &DB, job: &MiniCompletedJob, diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs index f36a5e6fc9..1126c94d17 100644 --- a/backend/windmill-worker/src/universal_pkg_installer.rs +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -333,7 +333,7 @@ pub async fn par_install_language_dependencies_all_at_once< mark_success(path.clone(), job_id, w_id).await; #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let language_name = _language_name.to_owned(); tokio::spawn(async move { if let Err(e) = crate::global_cache::build_tar_and_push( @@ -790,7 +790,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] let s3_pull_future = if is_not_pro { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { Some(crate::global_cache::pull_from_tar( os, dep.path.clone(), @@ -893,7 +893,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let language_name = _language_name.to_string(); let platform_agnostic = _platform_agnostic; let path = dep.path.clone(); @@ -954,8 +954,7 @@ async fn print_success( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if windmill_object_store::OBJECT_STORE_SETTINGS - .read() + if windmill_object_store::get_cache_object_store() .await .is_none() { diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index cecf0e316c..c188792a5e 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -46,6 +46,10 @@ pub enum WacOutput { form: Option, #[serde(default)] self_approval_disabled: Option, + #[serde(default)] + skin: Option, + #[serde(default)] + description: Option, }, /// Server-side sleep — suspend the workflow for a duration without holding a worker. #[serde(rename = "sleep")] @@ -108,6 +112,11 @@ pub enum WacPark { /// completes a job without a worker-measured duration — a cancel, the child-failure /// handler — falls back to `now() - started_at`. Left pointing at the first segment, /// that fallback reports the whole sleep or approval wait as execution time. +/// +/// Call it before any write to the parent's `v2_job_status` row in the same +/// transaction: a child's completion locks the queue row and then the status row +/// (`record_child_completion`), and taking them the other way round here can +/// deadlock against a stale child finishing while the parent re-dispatches. pub async fn suspend_wac_parent( tx: &mut Transaction<'_, Postgres>, job_id: &Uuid, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7e620439b0..56b4b26fb0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3405,7 +3405,7 @@ pub async fn run_worker( let suspend_first = suspend_first_success || rand::random::() < likelihood_of_suspend || last_suspend_first.elapsed().as_secs_f64() > 5.0 - || crate::result_processor::WAC_SUSPEND_READY + || windmill_common::wac::WAC_SUSPEND_READY .swap(false, Ordering::Relaxed); if suspend_first { @@ -3867,7 +3867,8 @@ pub async fn run_worker( let job_result = windmill_common::log_context::with_log_context( log_ctx, async { - let result = handle_queued_job( + // Keep large job-phase futures boxed to limit debug polling frames. + let result = Box::pin(handle_queued_job( arc_job.clone(), raw_code, raw_lock, @@ -3888,7 +3889,7 @@ pub async fn run_worker( flow_runners, #[cfg(feature = "benchmark")] &mut bench, - ) + )) .await; record_job_span_status(&result); result @@ -4644,7 +4645,7 @@ pub async fn handle_queued_job( let cached_res_path = if job.cache_ttl.is_some() { match conn { Connection::Sql(db) => { - Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await) + Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await?) } Connection::Http(_) => None, } @@ -5159,7 +5160,7 @@ async fn try_validate_schema( code, language, job.script_entrypoint_override.clone(), - )? { + ).await? { Ok(Some(schema_validator_from_main_arg_sig(&sig))) } else { Err(anyhow!("Job was expected to validate the arguments schema, but no schema was provided and couldn't be inferred from the script for language `{language:?}`. Try removing schema validation for this job").into()) @@ -5527,7 +5528,7 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); - run_language_executor( + let result = Box::pin(run_language_executor( job, conn, client, @@ -5552,8 +5553,111 @@ async fn handle_code_execution_job( &modules, false, in_pipeline, - ) - .await + )) + .await; + record_declared_warehouse_write(job, conn, code, &result).await; + result +} + +/// Record the outcome of a `// materialize manual dbt:////` +/// declaration, the way the DuckDB executor records a DuckLake target. +/// +/// Nothing generates warehouse DDL, so the script issues its own write and this +/// is the only thing that turns it into a `materialized_partition` row — the +/// relation's last writer on the run page and the graph. Language-agnostic on +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. +/// +/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` +/// rows, not from this one, so a lost row costs the relation its last writer and +/// nothing else. It must not fail a job whose write already landed. +/// +/// Shares the reach of every other runtime pipeline annotation, which is this +/// function's caller: a job handed to a dedicated worker or a flow runner never +/// passes through it, so — exactly as `// partitioned` is not resolved there — +/// such a run performs its write and records no row. +async fn record_declared_warehouse_write( + job: &MiniPulledJob, + conn: &Connection, + code: &str, + result: &error::Result>, +) { + use windmill_common::materialization::{ + MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED, + }; + // A DEPLOYED script only. The annotation is a deploy-time contract — `manual`, + // a three-segment relation, a configured warehouse — checked in + // `create_script_internal`, which also required write access to the path. A + // preview, hub or inline-flow body reaches this function without any of that, + // so honouring it there would let `jobs:run` alone restamp any relation's last + // writer from a script that never touched it. + if job.kind != JobKind::Script { + return; + } + // Cheap guard: the annotation scan is skipped for the overwhelming majority + // of jobs, which carry no `materialize` line at all. + if !code.contains("materialize") { + return; + } + let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code) + .materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt) + else { + return; + }; + // The slice this run wrote, resolved once upstream (`resolve_partition_for_job`) + // and carried in the args the cascade reads too, so a partitioned producer + // records the same identity everything else propagates. + let partition = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or_else(|| UNPARTITIONED.to_string()); + let (status, error) = match result { + Ok(_) => (MaterializationStatus::Materialized, None), + Err(e) => (MaterializationStatus::Failed, Some(e.to_string())), + }; + let recorded = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + &job.workspace_id, + windmill_common::assets::AssetKind::Dbt, + &m.target_path, + &partition, + status, + None, + None, + Some(job.id), + error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http( + client, + &job.workspace_id, + &RecordMaterializationRequest { + asset_kind: windmill_common::assets::AssetKind::Dbt, + asset_path: m.target_path.clone(), + partition, + status, + snapshot_id: None, + row_count: None, + job_id: Some(job.id), + error, + schema: None, + }, + ) + .await + } + }; + if let Err(e) = recorded { + tracing::warn!( + "recording the materialization of dbt://{} failed: {e:#}", + m.target_path + ); + } } /// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot @@ -6309,7 +6413,8 @@ mount {{ | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Nativets - | ScriptLang::Go => "//", + | ScriptLang::Go + | ScriptLang::Php => "//", _ => "", }; let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); @@ -6362,7 +6467,7 @@ mount {{ .await; if let Connection::Sql(db) = conn { - volume_setup = crate::volume_oss::setup_volumes_sql_worker( + volume_setup = Box::pin(crate::volume_oss::setup_volumes_sql_worker( &volume_mounts, db, &job.workspace_id, @@ -6375,10 +6480,10 @@ mount {{ language, &mut envs, &mut shared_mount, - ) + )) .await?; } else if let Connection::Http(http) = conn { - volume_setup = crate::volume_oss::setup_volumes_http_worker( + volume_setup = Box::pin(crate::volume_oss::setup_volumes_http_worker( &volume_mounts, http, &job.workspace_id, @@ -6391,7 +6496,7 @@ mount {{ language, &mut envs, &mut shared_mount, - ) + )) .await?; } } @@ -6887,7 +6992,7 @@ mount {{ if let Some(ref vol_client) = volume_setup.client { if let Connection::Sql(db) = conn { - crate::volume_oss::sync_volumes_sql_worker( + Box::pin(crate::volume_oss::sync_volumes_sql_worker( &volume_setup.states, &volume_setup.writable, vol_client, @@ -6897,13 +7002,13 @@ mount {{ worker_name, conn, result.is_ok(), - ) + )) .await; } } if let Connection::Http(http) = conn { - crate::volume_oss::sync_volumes_http_worker( + Box::pin(crate::volume_oss::sync_volumes_http_worker( &volume_setup.states, &volume_setup.writable, http, @@ -6912,7 +7017,7 @@ mount {{ worker_name, conn, result.is_ok(), - ) + )) .await; } @@ -6947,7 +7052,7 @@ mount {{ result } -pub fn parse_sig_of_lang( +pub async fn parse_sig_of_lang( code: &str, language: Option<&ScriptLang>, main_override: Option, @@ -6982,10 +7087,9 @@ pub fn parse_sig_of_lang( ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?), ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?), #[cfg(feature = "php")] - ScriptLang::Php => Some(windmill_parser_php::parse_php_signature( - code, - main_override, - )?), + ScriptLang::Php => { + Some(crate::php_executor::parse_php_signature(code, main_override).await?) + } #[cfg(not(feature = "php"))] ScriptLang::Php => None, #[cfg(feature = "rust")] diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 34f46307cf..ed543bd517 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2005,7 +2005,8 @@ pub async fn update_flow_status_after_job_completion_internal( if flow_job.cache_ttl.is_some() && success { let flow = RawData::Flow(flow_data.clone()); - let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await; + let cached_res_path = + cached_result_path(db, client, &flow_job, Some(&flow)).await?; save_in_cache( db, @@ -2034,8 +2035,8 @@ pub async fn update_flow_status_after_job_completion_internal( chat_ai_info.conversation_id, ) .await?; - let (duration, wac_job_ids) = if success { - let (_, duration, wac_job_ids) = add_completed_job( + let duration = if success { + let (_, duration) = add_completed_job( db, &cflow_job, true, @@ -2049,9 +2050,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - (duration, wac_job_ids) + duration } else { - let (_, duration, wac_job_ids) = add_completed_job( + let (_, duration) = add_completed_job( db, &cflow_job, false, @@ -2069,30 +2070,11 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - (duration, wac_job_ids) + duration }; flow_job_duration = flow_job .started_at .map(|x| FlowJobDuration { started_at: x, duration_ms: duration }); - - // If this flow is a WAC child (not a flow step, has parent), - // notify the WAC parent of completion. - if !flow_job.is_flow_step() { - if let Some(parent_job) = flow_job.parent_job { - if let Some(job_ids) = wac_job_ids { - let _ = crate::result_processor::handle_wac_child_completion( - db, - &flow_job.id, - parent_job, - &flow_job.workspace_id, - nresult.clone(), - success, - job_ids, - ) - .await; - } - } - } } true } else { @@ -3891,7 +3873,18 @@ async fn push_next_flow_job( drop(resume_messages); - let is_skipped = if let Some(skip_if) = &module.skip_if { + // `skip_if` is a one-time entry gate, so only first-entry statuses evaluate it. + // Once the module is looping, the last completed job is an inner iteration, not + // `previous_id`'s, and re-evaluating would alias `results.` to it. + // A restart-at-iteration also enters as `InProgress`: it resumes without re-gating. + let is_skipped = if let Some(skip_if) = module.skip_if.as_ref().filter(|_| { + matches!( + status_module, + FlowStatusModule::WaitingForPriorSteps { .. } + | FlowStatusModule::WaitingForEvents { .. } + | FlowStatusModule::WaitingForExecutor { .. } + ) + }) { let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status); let skip_if_res = compute_bool_from_expr( &skip_if.expr, @@ -4438,6 +4431,8 @@ async fn push_next_flow_job( .as_deref() .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) { + // A step with its own on-behalf-of carries a cached dispatch address, up to one + // notify poll stale; accepted, see `get_email_from_permissioned_as`. let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; check_tag_available_for_workspace_internal( db, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 9bba9a99e6..03150beb00 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -174,7 +174,7 @@ async fn handle_build_binary_job( if !crate::global_cache::object_store_available().await { return Ok(to_raw_value_owned(json!({ "status": "skipped", - "reason": "this worker cannot reach the instance object store, so the binary \ + "reason": "this worker cannot reach an object store, so the binary \ would not be shared with other workers", }))); } @@ -1795,6 +1795,17 @@ async fn lock_modules( agent, tool_inputs, } => { + if let Some(agent_path) = agent.as_deref().filter(|_| !skip_flow_update) { + sqlx::query!( + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + job_path, + agent_path, + job.workspace_id, + ) + .execute(db) + .await?; + } + // Extract FlowModules from tools and track their original indices // MCP tools don't need locking, so we filter them out let mut flow_modules = Vec::new(); diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 12930e8de4..9159434fd0 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.804.0"; +export const VERSION = "v1.812.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/.gitignore b/chat-sdk/.gitignore new file mode 100644 index 0000000000..1eae0cf670 --- /dev/null +++ b/chat-sdk/.gitignore @@ -0,0 +1,2 @@ +dist/ +node_modules/ diff --git a/chat-sdk/README.md b/chat-sdk/README.md new file mode 100644 index 0000000000..a233425d14 --- /dev/null +++ b/chat-sdk/README.md @@ -0,0 +1,286 @@ +# windmill-chat + +Build a chat interface on a Windmill flow deployed in **chat mode**, from any frontend +or from a Windmill raw app. The library is headless: it runs the flow, follows the +answer as it streams, keeps the conversation history, and hands you state to render. + +``` +npm install windmill-chat +``` + +No runtime dependencies. Optional peers: `react` for `windmill-chat/react`, `ai` for +`windmill-chat/ai-sdk`, `@assistant-ui/react` for `windmill-chat/assistant-ui`. + +| You build the UI with | Import | You get | +|---|---|---| +| Vercel AI SDK `useChat`, AI Elements | `windmill-chat/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | +| assistant-ui | `windmill-chat/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | +| Your own components | `windmill-chat/react` or `windmill-chat` | a hook / a store with messages, status and actions | + +## The flow + +Any deployed flow with **Chat mode** enabled in its settings works. Windmill passes the +message as the `user_message` input and threads the conversation through `memory_id`, +so an AI agent step remembers earlier turns. The answer is: + +- what the last step streams, when it is an AI agent step; +- otherwise the flow's result: its `windmill_chat_answer` field when it has one, a + string as is, anything else as JSON. + +## Vercel AI SDK (`useChat`, AI Elements) + +```tsx +import { useChat } from '@ai-sdk/react' +import { createWindmillChatTransport } from 'windmill-chat/ai-sdk' + +const transport = createWindmillChatTransport({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) +}) + +export function Support() { + const { messages, status, sendMessage, stop } = useChat({ id: conversationId, transport }) + // render `messages[i].parts`: text, reasoning and dynamic-tool parts, as with any AI SDK backend +} +``` + +The chat `id` is the conversation: reuse it to continue one, and pass a UUID when you +also read server history, so it matches what `flow_conversations` stores (any other id +maps to a fixed UUID). `sendMessage(msg, { body })` sends extra flow inputs. Tool calls +arrive as `dynamic-tool` parts (`input-available → output-available | output-error`), +which AI Elements' `` renders as is. A failed flow surfaces as `error`. +`regenerate()` runs the flow again with the same message: a new turn on the server, +not a replacement of the previous answer. + +The transport also carries the history helpers: `transport.loadMessages(id)` returns +`UIMessage`s for `useChat({ messages })` or `setMessages`, `transport.listConversations()` +and `transport.deleteConversation(id)`. Attachments are not supported: `sendMessage` with +`files` is refused with an explanatory error. + +## assistant-ui + +```tsx +import { AssistantRuntimeProvider } from '@assistant-ui/react' +import { useWindmillRuntime } from 'windmill-chat/assistant-ui' + +export function Support() { + const runtime = useWindmillRuntime({ baseUrl, workspace, flowPath, token }) + return ( + + {/* your assistant-ui components, thread list included */} + + ) +} +``` + +Conversations are threads: `ThreadListPrimitive` switches, creates and deletes them. +Tool calls render through your `tools` components (`MessagePrimitive.Parts`), reasoning +through `Reasoning`. It takes the same options as `useWindmillChat` below. + +## React + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +export function Support() { + const chat = useWindmillChat({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) + }) + const [draft, setDraft] = useState('') + + return ( +
+ {chat.messages.map((m) => ( +

+ {m.content} +

+ ))} +
{ + e.preventDefault() + chat.sendMessage(draft) + setDraft('') + }} + > + setDraft(e.target.value)} /> + + {chat.status === 'streaming' && ( + + )} + +
+ ) +} +``` + +The hook returns the [state](#state) plus the chat's methods. It recreates the chat +(fresh state, old one destroyed) when `flowPath`, `baseUrl`, `workspace`, `history`, +`storageKey` or the credential change: a different token string, or a switch between +no token, a string and a function; and when a `run` callback appears or goes away. +A token function is called through a ref, so passing a new closure on every render +is fine and never resets the chat, and so are `run` and the callbacks; when users +sign in and out behind a token function, change `storageKey` (their id) so local +history and state start over with them. + +## Raw apps + +Inside a Windmill raw app nothing needs configuring: the chat runs as the viewer, +against the Windmill the app is served from. + +```tsx +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +``` + +- **Unsandboxed app** (the default): the viewer's session is used. Viewers need + permission to run the flow. +- **Sandboxed app**: declare `jobs:run` in the app's frontend SDK scopes, and + `flow_conversations:write` for server-side history. The viewer consents once and + the app receives a token restricted to those scopes. +- **`wmill app dev`**: there is no viewer session on the dev server, so pass + `baseUrl`, `workspace` and `token` explicitly during development. + +## Any framework + +`createChat` returns a store: `subscribe` calls the listener immediately and on every +change, and returns the unsubscribe function. That is the Svelte store contract, so +`$chat` works as is; other frameworks wrap it in a few lines. + +```ts +import { createChat } from 'windmill-chat' + +const chat = createChat({ baseUrl, workspace, flowPath, token }) +chat.subscribe((state) => render(state)) +await chat.sendMessage('Hello') +``` + +```svelte + + +{#each $chat.messages as m (m.id)} +

{m.content}

+{/each} + +``` + +## Options + +| Option | | +|---|---| +| `flowPath` | Path of the deployed flow, e.g. `f/support/assistant`. Required. | +| `baseUrl` | The Windmill origin. Detected inside a raw app. | +| `workspace` | Detected inside a raw app. | +| `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. | +| `history` | `'server'`, `'local'` or `'none'`, see [History](#history). Defaults to `'server'` with a viewer session and `'local'` with an explicit `token`. | +| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones. | +| `storageKey` | Namespace for `local` history, e.g. the signed-in user's id. Local history is per browser and per flow; without it, users sharing a browser share it. | +| `fetch`, `storage` | Replacements for the globals, for tests and unusual runtimes. | +| `pageSize` | Messages and conversations per page of server history. Default 50. | +| `pollDelayMs` | How often, in ms, the server polls a running turn for the stream (Enterprise; 50 at the fastest, other servers ignore it). Unset, the server relaxes from 100 ms to 3 s over a long turn; set it when tokens must keep flowing at that pace. | +| `onFinish`, `onError` | Called when a turn has its answer, or could not run at all. | +| `run` | Runs the flow for a turn yourself and returns the job id, instead of the deployed flow at `flowPath` (Windmill's editor chats with an undeployed flow through a preview run this way). Pass `memory_id` = the conversation id. | + +## State + +```ts +interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: 'idle' | 'submitted' | 'streaming' | 'error' + error: Error | undefined + conversations: Conversation[] + history: 'server' | 'local' | 'none' + loadingMessages: boolean + hasMoreMessages: boolean +} + +interface ChatMessage { + id: string + role: 'user' | 'assistant' | 'tool' | 'system' + content: string + reasoning?: string // the model's reasoning summary, when streamed + tool?: { callId?: string; name: string; arguments?: string; result?: string; status: 'running' | 'success' | 'error' } + success: boolean // false for a failed flow or tool + pending: boolean // still streaming, or not yet confirmed by the server + createdAt: string + jobId?: string + stepName?: string + serverId?: string // the persisted row; `id` itself never changes, so list keys are stable +} +``` + +A turn goes `submitted` (the flow is queued) → `streaming` (the answer is arriving) → +`idle`. Tool calls appear as `tool` messages whose `status` moves from `running` to +`success` or `error`. A flow that fails still completes the turn: its error is the +answer, an `assistant` message with `success: false`. `status: 'error'` (with `error` +set) means the turn could not run or be followed at all, such as a refused request. + +Methods: `sendMessage(text, { inputs? })`, `stop()`, `newConversation()`, +`selectConversation(id)`, `loadConversations({ page?, perPage? })`, +`deleteConversation(id)`, `loadOlderMessages()`, `destroy()`. Switching conversations +stops following the current answer; the flow keeps running and, with server history, +its answer is there when you come back. + +## History + +Windmill stores every conversation of a chat-mode flow, and each Windmill user sees +only their own. `history: 'server'` reads that store: `loadConversations()` lists +them, `selectConversation(id)` loads one, `loadOlderMessages()` pages back. Every +message rendered from the server carries its `jobId` and `stepName`. + +That store is keyed by the **Windmill user**, so it fits a viewer session or a token +issued per user. With one token shared by every visitor of a site, all visitors would +see each other's conversations. For that setup use `history: 'local'` (the default +with an explicit `token`): the conversation list and messages stay in the browser's +`localStorage`, per Windmill instance, workspace and flow. `'none'` keeps nothing +beyond the page. + +When the default `'server'` mode turns out unreadable (a token or sandboxed app +without `flow_conversations` scopes), the chat switches itself to `'local'` and +`state.history` says so. Passing `history` explicitly disables that fallback. + +## Tokens + +Anything a browser holds can be read by its user, so give a chat token exactly what +the chat needs: + +| Setup | Scopes | +|---|---| +| Public site, one token for everyone | `jobs:run:flows:f/support/assistant`, and `history: 'local'`. The token can run that one flow and follow its jobs, nothing else. | +| Per-user tokens minted by your backend | The above plus `flow_conversations:write`, with `history: 'server'` (an explicit token defaults to local history). Return them from an endpoint and pass `token: () => fetch(...)`. | +| A Windmill user in the browser (raw app, embedded Windmill) | No token: the session is used. | + +The token's user must be allowed to run the flow. `stop()` closes the stream in any +case; cancelling the run on the server as well needs `jobs:write`, which also lets the +token read every job its user can see, so leave it out unless that matters. + +Anyone holding the token can run the flow with inputs of their choosing, so a flow +exposed this way should treat `user_message` and the other inputs as untrusted. + +## Lower level + +`WindmillChatApi` wraps the endpoints (`runFlow`, `streamJob`, `listConversations`, +`listMessages`, `deleteConversation`, `cancelJob`), `followJob` follows a run to +completion across the server's stream timeouts, `parseStreamEvents` decodes the AI +agent stream, `extractChatAnswer` turns a flow result into the text a chat shows, and +`conversationIdFor` maps any chat id to its conversation UUID. They are exported for +custom integrations. + +## For AI coding agents + +When asked to add a chat over a Windmill flow: the flow must be deployed with chat mode +on. Pick the entry point from the table at the top (`useChat` → `windmill-chat/ai-sdk`, +assistant-ui → `windmill-chat/assistant-ui`, otherwise `windmill-chat/react`). Inside a +Windmill raw app pass only `flowPath`. Elsewhere pass `baseUrl`, `workspace` and a +`token`; for a public page use a token scoped to `jobs:run:flows:` and leave +`history` at its default. Render `role`, `content`, `pending`, `success` and +`tool.status`; never build the SSE handling yourself. diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json new file mode 100644 index 0000000000..2fe22bcfbc --- /dev/null +++ b/chat-sdk/package-lock.json @@ -0,0 +1,3433 @@ +{ + "name": "windmill-chat", + "version": "1.812.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "windmill-chat", + "version": "1.812.0", + "license": "Apache-2.0", + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.80", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.80.tgz", + "integrity": "sha512-6t+07o8lSthpKf64Xb1qHWR2bWvJ3Fd2oFvS9fQc45p31bi2OUqan246e/ojAmZpWCiMPjKyQ4TBr4MYytnTiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/mcp": { + "version": "2.0.49", + "resolved": "https://registry.npmjs.org/@ai-sdk/mcp/-/mcp-2.0.49.tgz", + "integrity": "sha512-dD8//65te2C5B4UzPNHGLR7CStR5IvY7o7kXz0YYtPQs+aCdXc7Jb3fuvZcovShfP1KGvPIMTfXhp6Uz+UghJQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "cross-spawn": "^7.0.6", + "pkce-challenge": "^5.0.1" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.14.tgz", + "integrity": "sha512-yukP2tbcQQErG5gLCMBGvpvb/rM3D3KlTKG6eKKOdNHLHqtNNDEeBxdYrY/JL+O76B2ig5dXY19H/f1HFSvRiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.40.tgz", + "integrity": "sha512-zsXPwSAQ9mRJ2hvyITaLOYUyuGrBmzFhJXOg4mllGmla1PfNxZcm4GwqMiV2xQaDgcgBMdUGxXkp9xekZZNIkg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^7.29.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "4.0.102", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-4.0.102.tgz", + "integrity": "sha512-KOTRsaVUr6QeisktVqm57KfBMqBpxK9f2ay+defGnYsfvXpFE277zg107Q27LSpiTsDpmk2GHMqNWdU9bKaJSw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/mcp": "2.0.49", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "ai": "7.0.99", + "swr": "^2.4.1", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "react": "^18 || ~19.0.1 || ~19.1.2 || ^19.2.1" + } + }, + "node_modules/@assistant-ui/core": { + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@assistant-ui/core/-/core-0.3.18.tgz", + "integrity": "sha512-WR5/uqZuI6FNWogIe5EKmP7gdLhQhX38QqaAMAfH9DEPVzw40ON4sZC9vD47rpBj6yrXhHIsbt6jAGUKH4mASQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42", + "nanoid": "^6.0.1" + }, + "peerDependencies": { + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "assistant-cloud": "^0.2.0", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "assistant-cloud": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/react": { + "version": "0.15.19", + "resolved": "https://registry.npmjs.org/@assistant-ui/react/-/react-0.15.19.tgz", + "integrity": "sha512-+mEXA/ibBSoodj3LFUzRAdYZ+HvlrIg8uexPVjcY3ssksG1zHx8E9TpuwCDxg2h3XrYU8c/UZj/uFlI5tTWMoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@assistant-ui/core": "^0.3.18", + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "assistant-cloud": "^0.2.0", + "assistant-stream": "^0.3.42", + "radix-ui": "^1.6.7", + "react-textarea-autosize": "^8.5.9", + "safe-content-frame": "^0.0.30", + "zod": "^4.5.4", + "zustand": "^5.0.15" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/store": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@assistant-ui/store/-/store-0.3.13.tgz", + "integrity": "sha512-4u5YAMfjgr+jpJUWZ5f1uqXQuxvIiR8SRg1WWlFbIknk5S1Wli4Jp/nTQsOeXsIw0d7Zdb6FKiQw31gqs5z/Ew==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/tap": { + "version": "0.9.17", + "resolved": "https://registry.npmjs.org/@assistant-ui/tap/-/tap-0.9.17.tgz", + "integrity": "sha512-z53TiHiM3ai8XQ5B60Lg8IKCWM0XRf1Sz1jTQtqiuAATzVd6zeFedSHpxq4i5P0zi5uYVdN/dkGEpVTOyPc3ow==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@floating-ui/core": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "dev": true, + "license": "MIT" + }, + "node_modules/@happy-dom/global-registrator": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/@happy-dom/global-registrator/-/global-registrator-20.14.5.tgz", + "integrity": "sha512-B05ID9DhSwLs6mlm1fzlkAtTIvB3duCvjJjfr19LBrlTK7VZtRjDqoTRIVv13GuYuNdhByu8LSZgThwV3Rkj7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "happy-dom": "^20.14.5" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.149.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@quansync/fs": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/@radix-ui/number": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.3.tgz", + "integrity": "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.7.tgz", + "integrity": "sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/react-accessible-icon": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accessible-icon/-/react-accessible-icon-1.1.15.tgz", + "integrity": "sha512-WTQwcAvQf5sOcuUyi90lKPbhwcvQ+j55cjrSmeaN+L2vKU3DooOvlKw2MDeiJ5IkV5N905KW0/fGojKOBhD11A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-accordion": { + "version": "1.2.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.2.20.tgz", + "integrity": "sha512-jDhG9FvAEnlhnjrsINbNXcUa4G+L1KqSkJSunkbKEzFRcAb52jvM0PjPxPRvhe1HNc5F5yc0yzzWeeqlH4yBIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-alert-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-alert-dialog/-/react-alert-dialog-1.1.23.tgz", + "integrity": "sha512-VAYOiQRqj3GPpYJE0I9J+X8Ip05cyVlNdKOFeiGS2Ou1HHGfpl0BxOyZm6nmVDyU+W+NF3/XLzmjHmVGydhwgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-arrow": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.15.tgz", + "integrity": "sha512-v4zggRcjadnI+ClKDuijlQEW4tw3NoaeHc/PwpKnLoLLKNUG4InLegkstooLcRIUWCs+8L22dGURCVuFfOKfnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-aspect-ratio": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-aspect-ratio/-/react-aspect-ratio-1.1.15.tgz", + "integrity": "sha512-fy+dyVR+90nelK8rqIznFlxzx7uPcGbhxH8Nfr2bHb4UfSe+e3hklOC0luK0hDwVwnRX7xTRySpsrQVeW+/oNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-avatar": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-avatar/-/react-avatar-1.2.6.tgz", + "integrity": "sha512-4ULOTJ/mqy2hT9GlWa/MFHxHSvH3nJzHnZM1waNsc5Bonv7i70aNenghXmD97S6OJ81ekXONGGt4nT1r0PfEdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-checkbox": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.3.11.tgz", + "integrity": "sha512-Gnptr9pDDQxD3hgq2dtPbtrp/c2qH1mBwIzw3X/ivrMb2e1t0jMTi606fVEqFPaQR1ggXIVQWKj3P2WW9v7zGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collapsible": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.20.tgz", + "integrity": "sha512-mcGesGplBnzN2sbvJETzpCNfSMyPnb29q1GRLU+Ib7bJrpIG2ywmRoh2V5VbA2uNvKikKUlVbAPks7JDjz4A8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.15.tgz", + "integrity": "sha512-9W+B9NPF0NaaPh/1NJd3+KqsnlLqU9H7T2rvww+fp+T/evVXdNAyYcnfRQZFOjkR1ajQp3yORlqnI8soawLvNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz", + "integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.2.2.tgz", + "integrity": "sha512-RHCUGwKHDr0hDGg4X7ma4JG4/+12qxw8rkh5QKdDldlCvtja6nUx1Ef/8HVrJze81lEsgLQlqjzjGNHantgnQA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context-menu": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context-menu/-/react-context-menu-2.3.7.tgz", + "integrity": "sha512-CtXP35dxaB5T3zXSd+E3uHe/QpXcpYnZmxp6OaIbfthtfW4wyb77M23BG+bwIJDtsMwEP/YssdsmNyZu7jhWew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.23.tgz", + "integrity": "sha512-Ksw4WeROkO4rC9k/onilX/Ao2Cr1ku1unMNH+XSCcP4jSXYu7HDsg9n4ojMjVb22XpYjAQ9qfrFlVbru1vXDUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.4.tgz", + "integrity": "sha512-5pzg4FGQNpExhnhT2zlrP1wZFaYCd1K0nYWoFAdcYoYK868IEigqMX3B3f8yIoRlAhAeDWciLI6ZdCKHF9P4Vg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.19.tgz", + "integrity": "sha512-8g4pfOL9HoKKLWGiypT+dphVqjFfmcXO5GBnhsG6zI+lxAx/8feQpr+1LSN8Re3hiZ+XkLNS4O9ztK11/LzQ6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-effect-event": "0.0.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.24.tgz", + "integrity": "sha512-geq8l2rJkxvkXsT9RMgtUE3P8pITFpTsvYpbySi1IH4fZEABD/Gp85myayFgxk0ktljGMJnCbeFkyTusvSvv7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.6.tgz", + "integrity": "sha512-RNOJjfZMTyBM6xYmV3IVGXkPjIhcBAuv48POevAXwrGJhkWZ9p1rFoIS1JFooPuT193AZmRsCPhpoVJxx6OPoQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.16.tgz", + "integrity": "sha512-wmRZ2WWLvmt6KHy2rNPOdPUjwq5xOHY02+m+udwJTn0aNIox/rkskAvJTyTLGhPK6KgrUjlJUJpgmx/+wFiFIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-form": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-form/-/react-form-0.1.16.tgz", + "integrity": "sha512-Q4TLEn2A7TAypxwmd6R9EwrlXDvkfYSDMrq9/887AXAGh+G1rH+kYJKSTv+Si9Y0JPKTwKYv6PviAJosysNimA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-hover-card": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-hover-card/-/react-hover-card-1.1.23.tgz", + "integrity": "sha512-H8qONfZd3ltrU3+jHCIgITbWo6e1iTKvP9DHdrvYbX48ooRM5FjEDTn16AMwdfuOGkWdZEhpl3PLL/Wk/AnHDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz", + "integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label": { + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.1.15.tgz", + "integrity": "sha512-o/rdYEwZTTo5tjknnPeyQFU45kUC4i/XyeDPP+HGyi6XqpOP6Zf5Ya5vh/Yfe9Id5JiuWnnAx2XqIeD3UYZt0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.24.tgz", + "integrity": "sha512-uW7RVuU6Lp/ZtfeY4b3kL32zccgEWvPv1+cf17ubYzHa9cL8AHokmk36cG/XEiH/smbQvumnieXX9j/e9RqJWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menubar": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menubar/-/react-menubar-1.1.24.tgz", + "integrity": "sha512-eeVs0vf7cuqXaM0qLQCPcufImiJNVBXdJDLu7ZGYl2732UH23Qat/foNGrr6vYV3/DdTsBqASoggUFgH14OcZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-navigation-menu": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.2.22.tgz", + "integrity": "sha512-ou7iLEJ+yrhQndkkA4U21XIdS/CS45F4iXIkTZcb6/Ne9EMsOuDudVmCwmDnfFZZ+y1FZqXRNSIgBy+YMvZVZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-one-time-password-field": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-one-time-password-field/-/react-one-time-password-field-0.1.16.tgz", + "integrity": "sha512-Tj9P6ntAJEw52oq/F0AGknXR4XncxEt7XU47O3xJQOiWfLzEy3d9gtgKfvjSzGxzHkfL+VzvxGu2KTFsloJqXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-password-toggle-field": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-password-toggle-field/-/react-password-toggle-field-0.1.11.tgz", + "integrity": "sha512-4gvFnmDXu3dgj21CqsufzIameRvlRd4SBqaWhcrlrNhRo0Y5i/49AmRJYe1fdAM3G2VNBbmin4b0D6cdQocwgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popover": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.23.tgz", + "integrity": "sha512-mw58MrBlyHWFisTOYignD0vf/3gdcgAR+9of1s9G/38CbFiUwH1nCDkc0AUM9IrXFgN5Ue8n45j9WCgyM1sbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.7.tgz", + "integrity": "sha512-UsJrrd7w4wuKKTdvd/DNERVlwSlUcyXzjhyDwBk+3aPOsCjOY6ZSbxuw8E6lZTjjfP8Cpd0J8VVkrYUWyGYXyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-rect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.17.tgz", + "integrity": "sha512-vKQLcWypUnwZVvfV7UkGahH2g6ySe8M8R+zYBwPrv5byZ9QAW6cQVvNKo7GgmD+p8aYb6D9JBuvy8/WhOno2wQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.10.tgz", + "integrity": "sha512-3wyzCQ6+ubRA+D4uv9m95JYLXxmOHp05qjrkjeA7uKHHtjpPggQzc6DAb0URl7j67oR0K2foO4ip27TiX037Bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.10.tgz", + "integrity": "sha512-MucOnzh6hR5mid6VpkbglRAMYMjKLqRnGBbjXkzjK52fuQDd1qbkx78a5P40mkcnVXJdEVxm26E9OPAiUq7nBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.16.tgz", + "integrity": "sha512-5XnomAsoZZCY+KNTxbIghpGqPruZvKFNlvcAljVAOdDRDsH4/OZQxhtwo5wdtoDM5R6MhJBb2sPnDuRFep3lzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.4.7.tgz", + "integrity": "sha512-cgYFEkntCxppHZgtSZ+7vh0wbZQ+IC7PPMw8DSnRG27B6kDd32/Zw0OJt7dGDigCoprMuWHjg2PvUn3PYvPFoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.19.tgz", + "integrity": "sha512-V9jI6hDjT7l3jsCQD9bLNvDLM3tH/gdbOTp7Tefp3hbbgCGQoK7tUvrWiRlcoBHIZ809ElXwNQwVo0B98LuTXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.18.tgz", + "integrity": "sha512-Zn5Cd171wxsO3Dfg8HaW6RifTb9CYTKQJHs/G4+LN1GfmJpaQMZQyQxMprVPHpaz7QY4l9BxK2JwQuzHsXC8nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.7.tgz", + "integrity": "sha512-WFGImkmbzcfxeIwq/+4HvRN0pizBwbwQUED4I13ezQsDdfl38ZntN6TmR8XaSzPBqoCToe8rF75j6NPNDSzhbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-separator": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.15.tgz", + "integrity": "sha512-jOLO4lssEzWpoDu7G+Ze4VjwMRUBt291pnZD0gmalREZipnTX3wadQo7Fy48GCTfe14/YRN6rw/rOJqrE85Wxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slider": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.4.7.tgz", + "integrity": "sha512-mTSLf1GC/C0moWjTbvCM6Qn/gBjvlFt1azuWF2v7MN5C3Zq2U2J2lN3ZEYkpujuOU5Ro7A28wkviSxaKnG0BYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.3.3.tgz", + "integrity": "sha512-qx7oqnYbxnK9kYI9m317qmFmEgo6ywqWvbTogdj7cL9p3/yx4M48p7Rnw5z3H890cL/ow/EeWJsuTykeZVXP5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-switch": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.3.7.tgz", + "integrity": "sha512-48tB/4dn2UVLBCYhTu9AuR63IHl73l/qLbLgxd86noTUor4/K4LFDAcYjK+isP5313qxaFpjPVogE7+Y0/V3Kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.21.tgz", + "integrity": "sha512-UKxJlZid7FVtsk/WTxj4i4uSEgj2Au+KBbS7SQyTlzMhhn+86Cz3tISZdTa87bfEfcuvZezf2ZsxD4xuEKtkog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast": { + "version": "1.2.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.2.23.tgz", + "integrity": "sha512-ofhyAsYaocRGOs/n0XWdUOSVzEAG6BfrMVM8z0c0kLEWY38w/0WuMFPTJP/HVaZPYkMvHZoKIIhNcjbTCBILPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.18.tgz", + "integrity": "sha512-7lonPlKfSacd20GlOBx2ltuVKz9oqWYZz+oMQyOltw6t1y2nyftj2ZmwwUHYn49kqfDWcp8dNZm5NgV+5Z+mug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.1.19.tgz", + "integrity": "sha512-OtnwuSVjd1Ofi+AdnvhsjQdyuhCDwYs1w9RyB5BN/OavXOVQo42SYqQjwUnbPnaiPFBpQ9aX70dWeee+v2oBLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toolbar": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toolbar/-/react-toolbar-1.1.19.tgz", + "integrity": "sha512-Ph0IvtYw4VB12ZnZg+YtrGs8yJQsnizwo/zu0R4Y/nWugtJzA7Pg1eWeuDR9+LSqn+xjamss+UOSOJJJ4gx8jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-toggle-group": "1.1.19" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.2.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.2.16.tgz", + "integrity": "sha512-6EamKFRRnlpdadndbZ6LMwycfwkwPte1B42hs6QA0gYhjaOKqW4PZ4pjaW9UrlDX5eVt/OjncE7BFTPL5nmZhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.4.tgz", + "integrity": "sha512-R6OUY2e2fA6Yn6s+VSx5KBV6Nx8LQEhu+cz7LCej18rQ1HLyg9PSC9jP/ZNx0o6FAIK9c0F1kHylzSxKsdlkrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.6.tgz", + "integrity": "sha512-uEQJGT97ZA/TgP/Hydw47lHu+/vQj6z/0jA+WeTbK1o9Rx45GImjpD0tc3W5ad3D6XTSR6e1yEO0FvGq6WQfVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz", + "integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.5.tgz", + "integrity": "sha512-ge3ipobwSXTj4JyVtswQ7qZj0ZHdtbGuOno/LrgAAeSxtsJ6Vs4Gz5IkPH2bmqpjcLUFoqGhA/mueuIf63UXlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-is-hydrated": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-is-hydrated/-/react-use-is-hydrated-0.1.3.tgz", + "integrity": "sha512-umO/aJ+82CpOnhDZUTbILCQf7kU/g0iv+oGs/Q8jw7IkhWBzaEP4sA268PhFAJTFetbwp3ICc6ktpI4TqtxcIw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz", + "integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-previous": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.4.tgz", + "integrity": "sha512-XoSLhbRbqxFtgJoi2fNHA3C6pDlY34x508vUpUGoFZfvePfHXHbE1lC4FYFMnJWgiCRroSTw6fOsXQoVS9RwZg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.4.tgz", + "integrity": "sha512-cSOCh6JlkmfjLyNcLiu2nB4v+nm+dkZ+Q5KHWk/soo4U7ZLiEQFKHK9/YmtBHjfCEaU43IBKQOc4/uJmCaiCTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.4.tgz", + "integrity": "sha512-D3anSY15EJoxrihpsXI6SMrmmonnQtR2ni7arO+Lfdg3O95b9hNXxONk8jA5C8ANdF/h5HMAxejgs8PWJ6rlhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.11.tgz", + "integrity": "sha512-NFS86RYYZb4/exihaESBGOpMJFz8MGLAfu3mOBSGByVnVPC9JPASfYubxd/8KbkQK0sYAv8lVQDEQukDX/qXvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/rect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.3.tgz", + "integrity": "sha512-JtyZR+mqgBibTo8xea3B6ZRmzZiM/YeVBtUkas6zMuXjAlfIFIW2FgqeM9eLyvEaYX66vr6DJMK+4U6LV0KhNw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.8.tgz", + "integrity": "sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.8.tgz", + "integrity": "sha512-dIYTWl9XprMUiQFoc55KUyk/oS8SKYH3zFl0LTR7RT0Xj4hgSVyuJcroH8JUu8RcpF8fTB6E0aOwCkZoYPcDSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.8.tgz", + "integrity": "sha512-PCSDQGXD2IyTEFrcgPyBM8jJuGmrbCMuoIOXdbEGVemruKACXoLQJrb+A45Z0L5t1RQkdfJprAYPkikbh7dzdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.8.tgz", + "integrity": "sha512-Uk7lRsGhPFHVX/sAUC6D5H9Ol30dFHd6iquokll2th3LpdJ3F5CzQB+7DHn0Ri2mG+U7k2zXiPHDrwZenXhwSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.8.tgz", + "integrity": "sha512-DjszaTEVogPqA5bYzsEeqDCQxbcp2fexQwKcRspYji2yzR68fCf+e4fx6kBSRDwX5/brZaHw/hWS9+A/+/w9sQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.8.tgz", + "integrity": "sha512-zmwa7FTmdzB6aaEEuuls18H6Ap5JmJPSoPTuXixeJZV6tG40SyLkApQtz1g8ptZtiEKqj9OM0oNLPh1AgvE31Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.8.tgz", + "integrity": "sha512-KdYQDPHwJVnbFwdTGMgxsI9SqblBlz6STGM+w1We/d5B8OWWidYH0MwkU/uA1wM5fIpO2MkOVxXrNzzuZhw9ew==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.8.tgz", + "integrity": "sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.8.tgz", + "integrity": "sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.8.tgz", + "integrity": "sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.8.tgz", + "integrity": "sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.8.tgz", + "integrity": "sha512-xWBkPOF1Q9k/Gv1nQXnVdLxKu74jXppuOM4Z3mnypVUJJJwLsMl7hNJGRAUJoG8A5MgOI1ACKM+wBFxSJzKy4A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.8.tgz", + "integrity": "sha512-uz2ZvfgXbxqNwijjjbxrnvALwpyODDcgc1T1N8N3rf/DXKQmaFwmB4LX4yyjggpwN2obdQLb2rgirX5ffCWYng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/bun": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.4.2" + } + }, + "node_modules/@types/node": { + "version": "26.5.1", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, + "node_modules/@types/whatwg-mimetype": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-mimetype/-/whatwg-mimetype-3.0.2.tgz", + "integrity": "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.99", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.99.tgz", + "integrity": "sha512-Ov+3j/nSajaVH5hO8C94wN9wisG5tAJ8HWsLV9d/+nlzrRGUh3oYO3Kp1fRyUNWjAWLSpGLHLPaHCdfxb307Vg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.80", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ansis": { + "version": "4.4.0", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + } + }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/assistant-cloud": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/assistant-cloud/-/assistant-cloud-0.2.0.tgz", + "integrity": "sha512-LMvPaufIfZ0dpByMsNeuPFY6UeKQ1oiSBktUJuhjL/2eTZLSzzjIWumiLALlpdvUZbbJCRHy8j0ZgzlZJJlEfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.200.0", + "@opentelemetry/sdk-trace-base": "^2.1.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + } + } + }, + "node_modules/assistant-stream": { + "version": "0.3.42", + "resolved": "https://registry.npmjs.org/assistant-stream/-/assistant-stream-0.3.42.tgz", + "integrity": "sha512-5dQxc7XX92LuJ8wuHaOi/vkHitz2DARABzoE364O7o4jiePWalDpDpqU9bN+bXTvnSiRrpesy6t3Z3oDqRkdQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "nanoid": "^6.0.1", + "secure-json-parse": "^4.1.0" + }, + "peerDependencies": { + "ioredis": "^5.10.1 || ^6.0.0", + "redis": "^5.12.1" + }, + "peerDependenciesMeta": { + "ioredis": { + "optional": true + }, + "redis": { + "optional": true + } + } + }, + "node_modules/ast-kit": { + "version": "2.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "pathe": "^2.0.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/birpc": { + "version": "2.9.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/buffer-image-size": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/buffer-image-size/-/buffer-image-size-0.6.4.tgz", + "integrity": "sha512-nEh+kZOPY1w+gcCMobZ6ETUp9WfibndnosbpwB1iJk/8Gt5ZF2bhS6+B6bPYz424KtwsR6Rflc3tCz1/ghX2dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/bun-types": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/defu": { + "version": "6.1.7", + "dev": true, + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/diff": { + "version": "8.0.4", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dts-resolver": { + "version": "2.1.3", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "oxc-resolver": ">=11.0.0" + }, + "peerDependenciesMeta": { + "oxc-resolver": { + "optional": true + } + } + }, + "node_modules/empathic": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.3", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/happy-dom": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-20.14.5.tgz", + "integrity": "sha512-x/RzkpWO40bTjIoT30iQtt64FLLmH/iRcUCN2X//bLx7H3ifkdfPXyqsro/OYtqzIAhiLMMA7mmiOR9C3NOKjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "@types/whatwg-mimetype": "^3.0.2", + "@types/ws": "^8.18.1", + "buffer-image-size": "^0.6.4", + "entities": "^7.0.1", + "whatwg-mimetype": "^3.0.0", + "ws": "^8.21.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/hookable": { + "version": "5.5.3", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true, + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/ms": { + "version": "2.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-6.0.1.tgz", + "integrity": "sha512-3wVS3i51pE2pi1k5FFL/95BGfVS0kSsvDVuGXHOtxox/TywUmtgq+3qiTOTbs9J7KfHaXPiN171k/A6dBnaXFw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^22 || ^24 || >=26" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "dev": true, + "license": "MIT" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/quansync": { + "version": "1.0.0", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/radix-ui": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.6.7.tgz", + "integrity": "sha512-QBdhh1arIEUvPC0dQ5+nwWAxt7+N+oP/9jPwjJkGFoSk/sqxg32gJtSXGtFh8frAIcS6oC9cx2Q+7KYCQLOAeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-accessible-icon": "1.1.15", + "@radix-ui/react-accordion": "1.2.20", + "@radix-ui/react-alert-dialog": "1.1.23", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-aspect-ratio": "1.1.15", + "@radix-ui/react-avatar": "1.2.6", + "@radix-ui/react-checkbox": "1.3.11", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-context-menu": "2.3.7", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-dropdown-menu": "2.1.24", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-form": "0.1.16", + "@radix-ui/react-hover-card": "1.1.23", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-menubar": "1.1.24", + "@radix-ui/react-navigation-menu": "1.2.22", + "@radix-ui/react-one-time-password-field": "0.1.16", + "@radix-ui/react-password-toggle-field": "0.1.11", + "@radix-ui/react-popover": "1.1.23", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-progress": "1.1.16", + "@radix-ui/react-radio-group": "1.4.7", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-scroll-area": "1.2.18", + "@radix-ui/react-select": "2.3.7", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-slider": "1.4.7", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-switch": "1.3.7", + "@radix-ui/react-tabs": "1.1.21", + "@radix-ui/react-toast": "1.2.23", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-toggle-group": "1.1.19", + "@radix-ui/react-toolbar": "1.1.19", + "@radix-ui/react-tooltip": "1.2.16", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-escape-keydown": "1.1.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-textarea-autosize": { + "version": "8.5.9", + "resolved": "https://registry.npmjs.org/react-textarea-autosize/-/react-textarea-autosize-8.5.9.tgz", + "integrity": "sha512-U1DGlIQN5AwgjTyOEnI1oCcMuEr1pv1qOtklB2l4nyMGbHzWrI0eFsYK0zos2YWqAolJyG0IWJaqWmWj5ETh0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.20.13", + "use-composed-ref": "^1.3.0", + "use-latest": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/rolldown": { + "version": "1.2.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.149.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.8", + "@rolldown/binding-android-arm64": "1.2.8", + "@rolldown/binding-darwin-arm64": "1.2.8", + "@rolldown/binding-darwin-x64": "1.2.8", + "@rolldown/binding-freebsd-x64": "1.2.8", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.8", + "@rolldown/binding-linux-arm64-gnu": "1.2.8", + "@rolldown/binding-linux-arm64-musl": "1.2.8", + "@rolldown/binding-linux-ppc64-gnu": "1.2.8", + "@rolldown/binding-linux-s390x-gnu": "1.2.8", + "@rolldown/binding-linux-x64-gnu": "1.2.8", + "@rolldown/binding-linux-x64-musl": "1.2.8", + "@rolldown/binding-openharmony-arm64": "1.2.8", + "@rolldown/binding-win32-arm64-msvc": "1.2.8", + "@rolldown/binding-win32-x64-msvc": "1.2.8" + } + }, + "node_modules/rolldown-plugin-dts": { + "version": "0.13.14", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/generator": "^7.28.0", + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.1", + "ast-kit": "^2.1.1", + "birpc": "^2.5.0", + "debug": "^4.4.1", + "dts-resolver": "^2.1.1", + "get-tsconfig": "^4.10.1" + }, + "engines": { + "node": ">=20.18.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@typescript/native-preview": ">=7.0.0-dev.20250601.1", + "rolldown": "^1.0.0-beta.9", + "typescript": "^5.0.0", + "vue-tsc": "^2.2.0 || ^3.0.0" + }, + "peerDependenciesMeta": { + "@typescript/native-preview": { + "optional": true + }, + "typescript": { + "optional": true + }, + "vue-tsc": { + "optional": true + } + } + }, + "node_modules/safe-content-frame": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/safe-content-frame/-/safe-content-frame-0.0.30.tgz", + "integrity": "sha512-t8XO/59b+YaJCmCpJ3aTZ8TyB6LzG7VzCtUO3u8h0jUsK/wCvQPMro8ioCA6AQr0Ve8Cv9zunFp1VcFGm1aybg==", + "dev": true, + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/semver": { + "version": "7.8.5", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/swr": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.5.1.tgz", + "integrity": "sha512-BRw55e8r0B7SpDN20CAzoQAHl7y1yP7/Zt7oqUjMv0vSt2u2Xnkm88Ws+VypbV9BXHQVuSuyVq7zMjO16wSExw==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tsdown": { + "version": "0.12.9", + "dev": true, + "license": "MIT", + "dependencies": { + "ansis": "^4.1.0", + "cac": "^6.7.14", + "chokidar": "^4.0.3", + "debug": "^4.4.1", + "diff": "^8.0.2", + "empathic": "^2.0.0", + "hookable": "^5.5.3", + "rolldown": "^1.0.0-beta.19", + "rolldown-plugin-dts": "^0.13.12", + "semver": "^7.7.2", + "tinyexec": "^1.0.1", + "tinyglobby": "^0.2.14", + "unconfig": "^7.3.2" + }, + "bin": { + "tsdown": "dist/run.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@arethetypeswrong/core": "^0.18.1", + "publint": "^0.3.0", + "typescript": "^5.0.0", + "unplugin-lightningcss": "^0.4.0", + "unplugin-unused": "^0.5.0" + }, + "peerDependenciesMeta": { + "@arethetypeswrong/core": { + "optional": true + }, + "publint": { + "optional": true + }, + "typescript": { + "optional": true + }, + "unplugin-lightningcss": { + "optional": true + }, + "unplugin-unused": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unconfig": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "defu": "^6.1.4", + "jiti": "^2.6.1", + "quansync": "^1.0.0", + "unconfig-core": "7.5.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/unconfig-core": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "dev": true, + "license": "MIT" + }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-composed-ref": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/use-composed-ref/-/use-composed-ref-1.4.0.tgz", + "integrity": "sha512-djviaxuOOh7wkj0paeO1Q/4wMZ8Zrnag5H6yBvzN7AKKe8beOaED9SF5/ByLqsku8NP4zQqsvM2u3ew/tJK8/w==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-isomorphic-layout-effect": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.2.1.tgz", + "integrity": "sha512-tpZZ+EX0gaghDAiFR37hj5MgY6ZN55kLiPkJsKxBMZ6GZdOSPJXiOzPM984oPYZ5AnehYx5WQp1+ME8I/P/pRA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-latest": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/use-latest/-/use-latest-1.3.0.tgz", + "integrity": "sha512-mhg3xdm9NaM8q+gLT8KryJPnRFOz1/5XPBhmDEVZK1webPzDjrPk7f/mbpeLqTgB9msytYWANxgALOCJKnLvcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "use-isomorphic-layout-effect": "^1.1.1" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sync-external-store": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", + "integrity": "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zustand": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.15.tgz", + "integrity": "sha512-MpSEjRiBkA9crSYeOUH32rJC7SVqAbm0Fqcqge/bUi2PPoLcBWKOsG+C8mevmpr8TwXHBVkChbbJiyvkE+i/3A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.20.0" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "immer": ">=9.0.6", + "react": ">=18.0.0", + "use-sync-external-store": ">=1.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + }, + "use-sync-external-store": { + "optional": true + } + } + } + } +} diff --git a/chat-sdk/package.json b/chat-sdk/package.json new file mode 100644 index 0000000000..a5aff2c90b --- /dev/null +++ b/chat-sdk/package.json @@ -0,0 +1,108 @@ +{ + "name": "windmill-chat", + "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", + "version": "1.812.0", + "author": "Ruben Fiszel", + "license": "Apache-2.0", + "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/windmill-labs/windmill.git", + "directory": "chat-sdk" + }, + "bugs": { + "url": "https://github.com/windmill-labs/windmill/issues" + }, + "keywords": [ + "windmill", + "chat", + "ai", + "agent", + "react" + ], + "sideEffects": false, + "type": "module", + "main": "dist/index.cjs", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.cjs" + } + }, + "./react": { + "import": { + "types": "./dist/react.d.ts", + "default": "./dist/react.js" + }, + "require": { + "types": "./dist/react.d.ts", + "default": "./dist/react.cjs" + } + }, + "./ai-sdk": { + "import": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.js" + }, + "require": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.cjs" + } + }, + "./assistant-ui": { + "import": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.js" + }, + "require": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.cjs" + } + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsdown && tsc -p tsconfig.build.json", + "check": "tsc --noEmit", + "test": "bun test" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + }, + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + } +} diff --git a/chat-sdk/src/ai-sdk.ts b/chat-sdk/src/ai-sdk.ts new file mode 100644 index 0000000000..ba500446a3 --- /dev/null +++ b/chat-sdk/src/ai-sdk.ts @@ -0,0 +1,324 @@ +import type { ChatTransport, UIMessage, UIMessageChunk, UIMessagePart } from 'ai' +import { WindmillApiError, WindmillChatApi, type WindmillChatApiOptions } from './api' +import { followJob } from './follow' +import type { AgentStreamEvent } from './stream' +import type { ChatMessage, Conversation } from './types' +import { + conversationIdFor, + errorResultMessage, + extractChatAnswer, + isAbortError, + isErrorResult, + parseJsonOr, + randomId +} from './utils' + +export interface WindmillChatTransportOptions extends WindmillChatApiOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Extra flow inputs sent with every message; `sendMessage(msg, { body })` adds per-message ones. */ + inputs?: Record +} + +export interface WindmillChatTransport + extends ChatTransport { + /** The Windmill conversation id behind an AI SDK chat id (a UUID chat id is used as is). */ + conversationId(chatId: string): string + /** Server history of a chat as `UIMessage`s, oldest first, for `useChat({ messages })`. Needs `flow_conversations:read`. */ + loadMessages(chatId: string, options?: { page?: number; perPage?: number }): Promise + /** The user's conversations for this flow, most recent first. */ + listConversations(options?: { page?: number; perPage?: number }): Promise + deleteConversation(chatId: string): Promise +} + +interface JobEntry { + jobId: string + offset?: number + done: boolean +} + +/** + * A Vercel AI SDK `ChatTransport` over a chat-mode flow: `useChat({ transport })` + * (and AI Elements, which builds on it) then work against Windmill unchanged. + * The chat id is the conversation, so a UUID id lines up with server history. + */ +export function createWindmillChatTransport( + options: WindmillChatTransportOptions +): WindmillChatTransport { + const api = new WindmillChatApi(options) + // One in-flight or finished job per chat, for `reconnectToStream`. + const jobs = new Map() + + return { + conversationId: conversationIdFor, + + async sendMessages({ chatId, messages, abortSignal, body }) { + const last = messages[messages.length - 1] + if (!last || last.role !== 'user') { + throw new Error('windmill-chat: the last message must be a user message') + } + if (last.parts.some((p) => p.type === 'file')) { + throw new Error( + 'windmill-chat: attachments are not supported; upload the file yourself and pass its reference through `body`' + ) + } + const text = last.parts + .filter((p): p is Extract, { type: 'text' }> => p.type === 'text') + .map((p) => p.text) + .join('\n') + const memoryId = conversationIdFor(chatId) + const jobId = await api.runFlow( + options.flowPath, + { ...options.inputs, ...(body as Record | undefined), user_message: text }, + { memoryId, signal: abortSignal } + ) + const entry: JobEntry = { jobId, done: false } + jobs.set(chatId, entry) + return chunkStream(api, entry, abortSignal) + }, + + async reconnectToStream({ chatId, abortSignal }) { + const entry = jobs.get(chatId) + if (!entry || entry.done) return null + return chunkStream(api, entry, abortSignal) + }, + + async loadMessages(chatId, pagination) { + // A chat that hasn't sent anything yet has no conversation on the server. + const rows = await api.listMessages(conversationIdFor(chatId), pagination).catch((e) => { + if (e instanceof WindmillApiError && e.status === 404) return [] + throw e + }) + return toUIMessages( + rows.map((row) => ({ + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success: row.success ?? true, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + tool: toolFromRowContent(row.message_type, row.content, row.success ?? true) + })) + ) as UI_MESSAGE[] + }, + + async listConversations(pagination) { + const rows = await api.listConversations(options.flowPath, pagination) + return rows.map((row) => ({ + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + })) + }, + + async deleteConversation(chatId) { + await api.deleteConversation(conversationIdFor(chatId)) + jobs.delete(chatId) + } + } +} + +function toolFromRowContent(role: string, content: string, success: boolean): ChatMessage['tool'] { + if (role !== 'tool') return undefined + const name = /^Used (.+) tool$/.exec(content)?.[1] ?? /^Error executing (.+)$/.exec(content)?.[1] + return name ? { name, status: success ? 'success' : 'error' } : undefined +} + +/** Streams a job's answer as AI SDK chunks; resumes from `entry.offset` when the job is already running. */ +function chunkStream( + api: WindmillChatApi, + entry: JobEntry, + signal: AbortSignal | undefined +): ReadableStream { + return new ReadableStream({ + async start(controller) { + const parts = new PartWriter((chunk) => controller.enqueue(chunk)) + parts.emit({ type: 'start' }) + try { + let failure: string | undefined + for await (const event of followJob(api, entry.jobId, { + signal, + streamOffset: entry.offset, + onOffset: (offset) => { + entry.offset = offset + } + })) { + if (event.type === 'stream') { + for (const e of event.events) parts.apply(e) + continue + } + parts.closeOpen() + failure = await failureText(api, entry.jobId, event.result, signal) + if (failure === undefined && !parts.streamedText) { + // No agent streamed: the flow's result is the answer. + const answer = extractChatAnswer(event.result) + if (answer !== undefined) { + parts.text(answer) + parts.closeOpen() + } + } + } + entry.done = true + parts.emit(failure === undefined ? { type: 'finish' } : { type: 'error', errorText: failure }) + } catch (e) { + if (!isAbortError(e)) { + parts.closeOpen() + parts.emit({ type: 'error', errorText: e instanceof Error ? e.message : String(e) }) + } + } finally { + controller.close() + } + } + }) +} + +/** A completed flow's error, when the job did fail (the envelope alone is a legitimate result). */ +async function failureText( + api: WindmillChatApi, + jobId: string, + result: unknown, + signal: AbortSignal | undefined +): Promise { + if (!isErrorResult(result)) return undefined + const failed = await api + .getCompletedResult(jobId, signal) + .then((r) => r.success === false) + .catch(() => true) + return failed ? errorResultMessage(result) : undefined +} + +/** + * Turns agent events into AI SDK chunks. Text and reasoning are open parts that a + * tool call closes (a new round starts new parts); tool calls are `dynamic-tool` + * parts, since the UI declares no tools of its own. + */ +class PartWriter { + streamedText = false + #textId: string | undefined + #reasoningId: string | undefined + #started = new Set() + #inputSent = new Set() + + constructor(readonly emit: (chunk: UIMessageChunk) => void) {} + + text(delta: string): void { + this.streamedText = true + if (!this.#textId) { + this.#textId = randomId() + this.emit({ type: 'text-start', id: this.#textId }) + } + this.emit({ type: 'text-delta', id: this.#textId, delta }) + } + + reasoning(delta: string): void { + if (!this.#reasoningId) { + this.#reasoningId = randomId() + this.emit({ type: 'reasoning-start', id: this.#reasoningId }) + } + this.emit({ type: 'reasoning-delta', id: this.#reasoningId, delta }) + } + + closeOpen(): void { + if (this.#reasoningId) { + this.emit({ type: 'reasoning-end', id: this.#reasoningId }) + this.#reasoningId = undefined + } + if (this.#textId) { + this.emit({ type: 'text-end', id: this.#textId }) + this.#textId = undefined + } + } + + apply(event: AgentStreamEvent): void { + switch (event.type) { + case 'token_delta': + this.text(event.content) + break + case 'reasoning_token_delta': + this.reasoning(event.content) + break + case 'tool_call': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_call_arguments': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: parseJsonOr(event.arguments), + dynamic: true + }) + break + case 'tool_execution': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_result': + this.#toolStart(event.call_id, event.function_name) + if (!this.#inputSent.has(event.call_id)) { + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: undefined, + dynamic: true + }) + } + this.emit( + event.success + ? { type: 'tool-output-available', toolCallId: event.call_id, output: parseJsonOr(event.result), dynamic: true } + : { type: 'tool-output-error', toolCallId: event.call_id, errorText: event.result, dynamic: true } + ) + break + } + } + + #toolStart(callId: string, name: string): void { + if (this.#started.has(callId)) return + this.#started.add(callId) + this.emit({ type: 'tool-input-start', toolCallId: callId, toolName: name, dynamic: true }) + } +} + +/** + * `ChatMessage`s (Windmill's role-per-row model) as `UIMessage`s: an assistant + * turn becomes one message whose parts carry its text, reasoning and tool calls. + */ +export function toUIMessages(messages: ChatMessage[]): UIMessage[] { + const out: UIMessage[] = [] + for (const m of messages) { + if (m.role === 'user' || m.role === 'system') { + out.push({ id: m.id, role: m.role, parts: [{ type: 'text', text: m.content }] }) + continue + } + let target = out[out.length - 1] + if (!target || target.role !== 'assistant') { + target = { id: m.id, role: 'assistant', parts: [] } + out.push(target) + } + if (m.role === 'tool') { + const toolCallId = m.tool?.callId ?? m.id + const toolName = m.tool?.name ?? 'tool' + const input = parseJsonOr(m.tool?.arguments) + target.parts.push( + m.success + ? { type: 'dynamic-tool', toolName, toolCallId, state: 'output-available', input, output: parseJsonOr(m.tool?.result) ?? m.content } + : { type: 'dynamic-tool', toolName, toolCallId, state: 'output-error', input, errorText: m.tool?.result ?? m.content } + ) + continue + } + if (m.reasoning) target.parts.push({ type: 'reasoning', text: m.reasoning, state: 'done' }) + if (m.content) target.parts.push({ type: 'text', text: m.content, state: 'done' }) + } + return out +} diff --git a/chat-sdk/src/api.ts b/chat-sdk/src/api.ts new file mode 100644 index 0000000000..afc50ff0a1 --- /dev/null +++ b/chat-sdk/src/api.ts @@ -0,0 +1,301 @@ +import type { FetchLike, TokenSource } from './types' + +export interface WindmillChatApiOptions { + baseUrl: string + workspace: string + /** Omit to rely on the session cookie of the Windmill origin. */ + token?: TokenSource + fetch?: FetchLike + /** Server poll interval for a turn's stream (Enterprise; see `ChatOptions.pollDelayMs`). */ + pollDelayMs?: number +} + +export class WindmillApiError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WindmillApiError' + } +} + +export interface FlowConversation { + id: string + workspace_id: string + flow_path: string + title?: string | null + created_at: string + updated_at: string + created_by: string +} + +export interface FlowConversationMessage { + id: string + conversation_id: string + message_type: 'user' | 'assistant' | 'system' | 'tool' + content: string + job_id?: string | null + created_at: string + created_seq: number + step_name?: string | null + success?: boolean +} + +export type JobUpdateEvent = + | { + type: 'update' + running?: boolean + completed?: boolean + new_result_stream?: string + stream_offset?: number + only_result?: unknown + flow_stream_job_id?: string + } + | { type: 'error'; error: string } + | { type: 'notfound' } + | { type: 'timeout' } + | { type: 'ping' } + +export interface CompletedJobResult { + completed: boolean + success?: boolean + result?: unknown +} + +/** The part of a flow job's status that names the jobs its steps ran as. */ +export interface FlowJobStatus { + flow_status?: { + modules?: FlowStepStatus[] | null + failure_module?: FlowStepStatus | null + preprocessor_module?: FlowStepStatus | null + } | null +} + +export interface FlowStepStatus { + job?: string | null + flow_jobs?: string[] | null + /** An agent step's rounds; a tool call ran as a job of its own, which its row is persisted under. */ + agent_actions?: { type?: string; job_id?: string | null }[] | null +} + +/** Thin client over the Windmill endpoints a chat-mode flow uses. */ +export class WindmillChatApi { + readonly #baseUrl: string + readonly #workspace: string + readonly #token: TokenSource | undefined + readonly #fetch: FetchLike + readonly #pollDelayMs: number | undefined + + constructor(options: WindmillChatApiOptions) { + this.#baseUrl = normalizeBaseUrl(options.baseUrl) + this.#workspace = options.workspace + this.#token = options.token + this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init)) + this.#pollDelayMs = options.pollDelayMs + } + + /** Starts a turn: runs the flow with `memory_id` set to the conversation id. Returns the job id. */ + async runFlow( + flowPath: string, + args: Record, + options: { memoryId: string; signal?: AbortSignal } + ): Promise { + const res = await this.#request(`jobs/run/f/${encodePath(flowPath)}`, { + method: 'POST', + query: { memory_id: options.memoryId, skip_preprocessor: 'true' }, + body: args, + signal: options.signal + }) + return (await res.text()).trim() + } + + /** + * One server-sent-events connection to a job's updates. The server closes it after + * `TIMEOUT_SSE_STREAM` (a `timeout` event); resume by calling again with the last + * `stream_offset`, never by re-running the flow. + */ + async *streamJob( + jobId: string, + options: { streamOffset?: number; signal?: AbortSignal } = {} + ): AsyncGenerator { + const query: Record = { fast: 'true', only_result: 'true' } + if (this.#pollDelayMs !== undefined) query.poll_delay_ms = String(this.#pollDelayMs) + if (options.streamOffset !== undefined) { + query.stream_offset = String(options.streamOffset) + } + const res = await this.#request(`jobs_u/getupdate_sse/${encodeURIComponent(jobId)}`, { + query, + accept: 'text/event-stream', + signal: options.signal + }) + if (!res.body) { + throw new WindmillApiError('The job update stream has no body', res.status) + } + for await (const data of readServerSentEvents(res.body)) { + try { + yield JSON.parse(data) as JobUpdateEvent + } catch { + // A frame that isn't JSON carries nothing the chat can use. + } + } + } + + async getCompletedResult(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request( + `jobs_u/completed/get_result_maybe/${encodeURIComponent(jobId)}`, + { signal } + ) + return (await res.json()) as CompletedJobResult + } + + /** A flow job with its status: the step job ids are what persisted messages carry as `job_id`. */ + async getFlowJob(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request(`jobs_u/get/${encodeURIComponent(jobId)}`, { + query: { no_logs: 'true' }, + signal + }) + return (await res.json()) as FlowJobStatus + } + + async cancelJob(jobId: string, reason = 'Stopped from the chat'): Promise { + await this.#request(`jobs_u/queue/cancel/${encodeURIComponent(jobId)}`, { + method: 'POST', + body: { reason } + }) + } + + async listConversations( + flowPath: string, + options: { page?: number; perPage?: number; signal?: AbortSignal } = {} + ): Promise { + const res = await this.#request('flow_conversations/list', { + query: pagination(options, { flow_path: flowPath }), + signal: options.signal + }) + return (await res.json()) as FlowConversation[] + } + + /** + * Without `afterSeq`: one page counted from the newest message, returned oldest first. + * With `afterSeq`: the messages created after that cursor, oldest first. + */ + async listMessages( + conversationId: string, + options: { page?: number; perPage?: number; afterSeq?: number; signal?: AbortSignal } = {} + ): Promise { + const extra: Record = {} + if (options.afterSeq !== undefined) extra.after_seq = String(options.afterSeq) + const res = await this.#request( + `flow_conversations/${encodeURIComponent(conversationId)}/messages`, + { query: pagination(options, extra), signal: options.signal } + ) + return (await res.json()) as FlowConversationMessage[] + } + + async deleteConversation(conversationId: string): Promise { + await this.#request(`flow_conversations/delete/${encodeURIComponent(conversationId)}`, { + method: 'DELETE' + }) + } + + async #request( + path: string, + init: { + method?: string + query?: Record + body?: unknown + accept?: string + signal?: AbortSignal + } = {} + ): Promise { + const url = new URL(`${this.#baseUrl}/api/w/${encodeURIComponent(this.#workspace)}/${path}`) + for (const [k, v] of Object.entries(init.query ?? {})) url.searchParams.set(k, v) + + const headers: Record = {} + if (init.accept) headers['Accept'] = init.accept + if (init.body !== undefined) headers['Content-Type'] = 'application/json' + const token = typeof this.#token === 'function' ? await this.#token() : this.#token + if (token) headers['Authorization'] = `Bearer ${token}` + + const res = await this.#fetch(url.toString(), { + method: init.method ?? 'GET', + headers, + body: init.body === undefined ? undefined : JSON.stringify(init.body), + // A token must not be paired with ambient cookies; without one, the cookie is + // the credential and only rides same-origin requests. + credentials: token ? 'omit' : 'same-origin', + signal: init.signal + }) + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new WindmillApiError( + `${init.method ?? 'GET'} ${path} failed (${res.status})${text ? `: ${text}` : ''}`, + res.status + ) + } + return res + } +} + +export function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.replace(/\/+$/, '').replace(/\/api$/, '') +} + +function encodePath(path: string): string { + return path.split('/').map(encodeURIComponent).join('/') +} + +function pagination( + options: { page?: number; perPage?: number }, + extra: Record +): Record { + const query = { ...extra } + if (options.page !== undefined) query.page = String(options.page) + if (options.perPage !== undefined) query.per_page = String(options.perPage) + return query +} + +/** Yields the `data` payload of each event in a `text/event-stream` body. */ +export async function* readServerSentEvents( + body: ReadableStream +): AsyncGenerator { + const reader = body.getReader() + const decoder = new TextDecoder() + let buffer = '' + // A CR ending a chunk may be half of a CRLF; it waits for the next chunk. + let carry = '' + try { + while (true) { + const { value, done } = await reader.read() + if (done) break + let text = carry + decoder.decode(value, { stream: true }) + carry = '' + if (text.endsWith('\r')) { + carry = '\r' + text = text.slice(0, -1) + } + buffer += text.replace(/\r\n?/g, '\n') + let end: number + while ((end = buffer.indexOf('\n\n')) !== -1) { + const data = eventData(buffer.slice(0, end)) + buffer = buffer.slice(end + 2) + if (data !== undefined) yield data + } + } + if (carry) buffer += '\n' + const data = eventData(buffer) + if (data !== undefined) yield data + } finally { + // Closes the connection when the consumer stops early. + reader.cancel().catch(() => {}) + } +} + +function eventData(block: string): string | undefined { + const lines = block + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(line.startsWith('data: ') ? 6 : 5)) + return lines.length > 0 ? lines.join('\n') : undefined +} diff --git a/chat-sdk/src/assistant-ui.ts b/chat-sdk/src/assistant-ui.ts new file mode 100644 index 0000000000..ef22f28531 --- /dev/null +++ b/chat-sdk/src/assistant-ui.ts @@ -0,0 +1,124 @@ +import { + useExternalStoreRuntime, + type AppendMessage, + type AssistantRuntime, + type ExternalStoreAdapter, + type ThreadMessageLike +} from '@assistant-ui/react' +import { useEffect, useMemo } from 'react' +import { useWindmillChat } from './react' +import type { ChatMessage, ChatOptions } from './types' +import { parseJsonOr } from './utils' + +/** One assistant turn: the assistant and tool rows between two user messages. */ +export interface WindmillTurn { + id: string + role: 'user' | 'assistant' | 'system' + messages: ChatMessage[] +} + +export interface WindmillRuntimeOptions extends ChatOptions { + /** Load the conversation list on mount so `ThreadListPrimitive` has something to show. Default true. */ + threadList?: boolean +} + +/** + * An assistant-ui runtime over a chat-mode flow, for `AssistantRuntimeProvider`. + * Conversations become threads, so the thread list primitives switch, create and + * delete Windmill conversations. + */ +export function useWindmillRuntime(options: WindmillRuntimeOptions): AssistantRuntime { + const chat = useWindmillChat(options) + const turns = useMemo(() => groupTurns(chat.messages), [chat.messages]) + const withThreadList = options.threadList !== false + useEffect(() => { + if (withThreadList) chat.loadConversations().catch(() => {}) + }, [chat.chat, withThreadList]) + + const adapter: ExternalStoreAdapter = { + messages: turns, + isRunning: chat.status === 'submitted' || chat.status === 'streaming', + convertMessage: toThreadMessage, + onNew: async (message: AppendMessage) => { + if (message.role !== 'user') return + await chat.sendMessage(appendedText(message)) + }, + onCancel: async () => { + await chat.stop() + }, + adapters: withThreadList + ? { + threadList: { + threadId: chat.conversationId, + threads: chat.conversations.map((c) => ({ status: 'regular' as const, id: c.id, title: c.title })), + onSwitchToNewThread: () => chat.newConversation(), + onSwitchToThread: (id) => chat.selectConversation(id), + onDelete: (id) => chat.deleteConversation(id) + } + } + : undefined + } + return useExternalStoreRuntime(adapter) +} + +function appendedText(message: AppendMessage): string { + return message.content + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text) + .join('\n') +} + +/** Folds the role-per-row message list into turns: one entry per user message, one per answer. */ +export function groupTurns(messages: ChatMessage[]): WindmillTurn[] { + const turns: WindmillTurn[] = [] + for (const m of messages) { + const last = turns[turns.length - 1] + if (m.role === 'user' || m.role === 'system' || !last || last.role !== 'assistant') { + turns.push({ id: m.id, role: m.role === 'tool' ? 'assistant' : m.role, messages: [m] }) + } else { + last.messages.push(m) + } + } + return turns +} + +/** A turn as assistant-ui content parts; the status reflects streaming and a failed flow. */ +export function toThreadMessage(turn: WindmillTurn): ThreadMessageLike { + const first = turn.messages[0] + const createdAt = new Date(first.createdAt) + if (turn.role !== 'assistant') { + return { id: turn.id, role: turn.role, createdAt, content: [{ type: 'text', text: first.content }] } + } + const content: ThreadContentPart[] = [] + for (const m of turn.messages) { + if (m.role === 'tool') { + const args = parseJsonOr(m.tool?.arguments) + content.push({ + type: 'tool-call', + toolCallId: m.tool?.callId ?? m.id, + toolName: m.tool?.name ?? 'tool', + args: (isJsonObject(args) ? args : args === undefined ? {} : { input: args }) as ToolCallArgs, + argsText: m.tool?.arguments ?? '', + result: m.tool?.status === 'running' ? undefined : (parseJsonOr(m.tool?.result) ?? m.content), + isError: m.tool?.status === 'error' + }) + continue + } + if (m.reasoning) content.push({ type: 'reasoning', text: m.reasoning }) + if (m.content) content.push({ type: 'text', text: m.content }) + } + const last = turn.messages[turn.messages.length - 1] + const status: ThreadMessageLike['status'] = turn.messages.some((m) => m.pending) + ? { type: 'running' } + : last.role === 'assistant' && !last.success + ? { type: 'incomplete', reason: 'error', error: last.content } + : { type: 'complete', reason: 'stop' } + return { id: turn.id, role: 'assistant', createdAt, content, status } +} + +type ThreadContentPart = Exclude[number] +type ToolCallArgs = Extract['args'] + +function isJsonObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts new file mode 100644 index 0000000000..dda8f10d05 --- /dev/null +++ b/chat-sdk/src/chat.ts @@ -0,0 +1,761 @@ +import { + WindmillApiError, + WindmillChatApi, + type FlowConversation, + type FlowConversationMessage +} from './api' +import { resolveConfig, type ResolvedConfig } from './config' +import { followJob } from './follow' +import { createLocalHistory, type LocalHistory } from './history' +import type { AgentStreamEvent } from './stream' +import type { + Chat, + ChatMessage, + ChatOptions, + ChatState, + Conversation, + ToolInvocation +} from './types' +import { + conversationTitle, + errorResultMessage, + extractChatAnswer, + isAbortError, + isErrorResult, + now, + randomId, + sleep +} from './utils' + +const POLL_INTERVAL_MS = 1000 +/** Local history mirrors the state; a stream of deltas is coalesced into one write. */ +const PERSIST_DEBOUNCE_MS = 250 +/** Messages persist from spawned tasks that can land just after the flow completes. */ +const RECONCILE_ATTEMPTS = 3 +const RECONCILE_DELAY_MS = 400 + +interface Turn { + controller: AbortController + conversationId: string + /** Id of the turn's user message; the answer is whatever follows it. */ + userMessageId: string + jobId?: string + /** The flow job and its step jobs; a persisted answer carries one of them as `job_id`. */ + jobIds?: Set + /** Id of the streaming assistant message; cleared when a tool call ends the round. */ + assistantId?: string + streamedText: boolean +} + +export function createChat(options: ChatOptions): Chat { + return new ChatImpl(options) +} + +class ChatImpl implements Chat { + readonly #config: ResolvedConfig + readonly #api: WindmillChatApi + readonly #local: LocalHistory + readonly #listeners = new Set<(state: ChatState) => void>() + #state: ChatState + #turn: Turn | undefined + #page = 1 + #persistTimer: ReturnType | undefined + + constructor(options: ChatOptions) { + this.#config = resolveConfig(options) + this.#api = new WindmillChatApi({ + baseUrl: this.#config.baseUrl, + workspace: this.#config.workspace, + token: this.#config.token, + fetch: this.#config.fetch, + pollDelayMs: this.#config.pollDelayMs + }) + this.#local = createLocalHistory( + this.#config.storage, + `windmill-chat:${this.#config.baseUrl}:${this.#config.workspace}:${this.#config.flowPath}` + + (this.#config.storageKey ? `:${this.#config.storageKey}` : '') + ) + this.#state = { + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + conversations: [], + history: this.#config.history, + loadingMessages: false, + hasMoreMessages: false + } + } + + getState = (): ChatState => this.#state + + subscribe = (listener: (state: ChatState) => void): (() => void) => { + this.#listeners.add(listener) + listener(this.#state) + return () => { + this.#listeners.delete(listener) + } + } + + sendMessage = async ( + text: string, + options: { inputs?: Record } = {} + ): Promise => { + const content = text.trim() + if (!content) return + if (this.#turn) { + throw new Error('windmill-chat: a message is already being answered; call stop() first') + } + const isNew = this.#state.conversationId === undefined + const conversationId = this.#state.conversationId ?? randomId() + const turn: Turn = { + controller: new AbortController(), + conversationId, + userMessageId: `pending-${randomId()}`, + streamedText: false + } + this.#turn = turn + + const timestamp = now() + const conversation: Conversation = this.#state.conversations.find( + (c) => c.id === conversationId + ) ?? { id: conversationId, title: conversationTitle(content), createdAt: timestamp, updatedAt: timestamp } + const touched = { ...conversation, updatedAt: timestamp } + this.#set({ + conversationId, + conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)], + messages: [ + ...this.#state.messages, + { id: turn.userMessageId, role: 'user', content, success: true, createdAt: timestamp, pending: true } + ], + status: 'submitted', + error: undefined + }) + this.#rememberConversation() + + try { + const args = { ...this.#config.inputs, ...options.inputs, user_message: content } + const context = { memoryId: conversationId, conversationId, signal: turn.controller.signal } + turn.jobId = this.#config.run + ? await this.#config.run(args, context) + : await this.#api.runFlow(this.#config.flowPath, args, context) + const stopPolling = this.#state.history === 'server' ? this.#startPolling(turn) : () => {} + let result: unknown + try { + result = await this.#follow(turn, stopPolling) + } finally { + stopPolling() + } + await this.#finishTurn(turn, result, isNew) + } catch (e) { + // stop() and a conversation switch abort the turn and settle the state themselves. + if (turn.controller.signal.aborted || isAbortError(e)) return + this.#failTurn(turn, e) + } finally { + if (this.#turn === turn) this.#turn = undefined + } + } + + stop = async (): Promise => { + const turn = this.#turn + if (!turn) return + this.#detachTurn() + if (this.#state.conversationId === turn.conversationId) { + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + this.#persistLocal() + } + if (turn.jobId) { + // Needs `jobs:write` on a token; the stream is closed either way. + await this.#api.cancelJob(turn.jobId).catch(() => {}) + } + if (this.#state.history !== 'server') return + // A cancelled flow persists its failure as the assistant's answer. Picked up only + // while the conversation is still idle: a turn started meanwhile owns the state. + await sleep(RECONCILE_DELAY_MS).catch(() => {}) + if (this.#turn || this.#state.conversationId !== turn.conversationId) return + await this.#syncFromServer(turn.conversationId).catch(() => {}) + } + + newConversation = (): void => { + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: false, + hasMoreMessages: false + }) + } + + selectConversation = async (conversationId: string): Promise => { + if (conversationId === this.#state.conversationId) return + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: true, + hasMoreMessages: false + }) + if (this.#state.history !== 'server') { + this.#set({ + messages: this.#state.history === 'local' ? this.#local.getMessages(conversationId) : [], + loadingMessages: false + }) + return + } + try { + const rows = await this.#api.listMessages(conversationId, { + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + this.#set({ + messages: rows.map(fromRow), + loadingMessages: false, + hasMoreMessages: rows.length === this.#config.pageSize + }) + } catch (e) { + if (this.#state.conversationId !== conversationId) return + if (this.#fallBackToLocal(e)) { + this.#set({ messages: this.#local.getMessages(conversationId), loadingMessages: false }) + return + } + this.#set({ loadingMessages: false, status: 'error', error: toError(e) }) + } + } + + loadConversations = async ( + options: { page?: number; perPage?: number } = {} + ): Promise => { + const page = options.page ?? 1 + let conversations: Conversation[] + if (this.#state.history === 'server') { + try { + const rows = await this.#api.listConversations(this.#config.flowPath, { + page, + perPage: options.perPage ?? this.#config.pageSize + }) + conversations = rows.map(fromConversation) + } catch (e) { + if (!this.#fallBackToLocal(e)) throw e + conversations = this.#local.listConversations() + } + } else { + conversations = this.#state.history === 'local' ? this.#local.listConversations() : [] + } + const known = new Set(this.#state.conversations.map((c) => c.id)) + this.#set({ + conversations: + page === 1 + ? conversations + : [...this.#state.conversations, ...conversations.filter((c) => !known.has(c.id))] + }) + return conversations + } + + deleteConversation = async (conversationId: string): Promise => { + if (this.#state.conversationId === conversationId) { + // Nothing of the current turn may be written back under the deleted id. + this.#detachTurn() + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + this.newConversation() + } + if (this.#state.history === 'server') { + await this.#api.deleteConversation(conversationId) + } else if (this.#state.history === 'local') { + this.#local.deleteConversation(conversationId) + } + this.#set({ conversations: this.#state.conversations.filter((c) => c.id !== conversationId) }) + } + + loadOlderMessages = async (): Promise => { + const conversationId = this.#state.conversationId + if ( + !conversationId || + this.#state.history !== 'server' || + !this.#state.hasMoreMessages || + this.#state.loadingMessages + ) { + return + } + const page = this.#page + 1 + this.#set({ loadingMessages: true }) + try { + const rows = await this.#api.listMessages(conversationId, { + page, + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + const known = new Set(this.#state.messages.map((m) => m.serverId ?? m.id)) + this.#page = page + this.#set({ + messages: [...rows.map(fromRow).filter((m) => !known.has(m.id)), ...this.#state.messages], + hasMoreMessages: rows.length === this.#config.pageSize + }) + } finally { + if (this.#state.conversationId === conversationId) this.#set({ loadingMessages: false }) + } + } + + destroy = (): void => { + this.#leaveConversation() + } + + // ---- turn internals ---- + + async #follow(turn: Turn, onStreamStart: () => void): Promise { + let started = false + for await (const event of followJob(this.#api, turn.jobId!, { signal: turn.controller.signal })) { + if (event.type === 'completed') return event.result + if (!started) { + started = true + // Persisted rows for the streaming step would duplicate what is streaming. + onStreamStart() + } + this.#applyEvents(turn, event.events) + } + throw new Error('windmill-chat: the job stream ended before the flow completed') + } + + #applyEvents(turn: Turn, events: AgentStreamEvent[]): void { + if (events.length === 0 || !this.#turnActive(turn)) return + let messages = [...this.#state.messages] + const upsertTool = ( + callId: string, + name: string, + patch: Partial & { content?: string; success?: boolean } + ) => { + const { content, success, ...toolPatch } = patch + // Only this turn's tool messages are pending; a provider may reuse call ids across turns. + const i = messages.findIndex( + (m) => m.pending && m.role === 'tool' && m.tool?.callId === callId + ) + if (i >= 0) { + const existing = messages[i] + messages[i] = { + ...existing, + content: content ?? existing.content, + success: success ?? existing.success, + tool: { ...existing.tool!, ...toolPatch } + } + } else { + messages.push({ + id: `pending-${randomId()}`, + role: 'tool', + content: content ?? '', + success: success ?? true, + createdAt: now(), + pending: true, + tool: { callId, name, status: 'running', ...toolPatch } + }) + } + } + const appendAssistant = (text: string, reasoning: string) => { + const i = turn.assistantId + ? messages.findIndex((m) => m.id === turn.assistantId) + : -1 + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...m, + content: m.content + text, + reasoning: reasoning ? (m.reasoning ?? '') + reasoning : m.reasoning + } + } else { + turn.assistantId = `pending-${randomId()}` + messages.push({ + id: turn.assistantId, + role: 'assistant', + content: text, + reasoning: reasoning || undefined, + success: true, + createdAt: now(), + pending: true + }) + } + } + for (const event of events) { + switch (event.type) { + case 'token_delta': + turn.streamedText = true + appendAssistant(event.content, '') + break + case 'reasoning_token_delta': + appendAssistant('', event.content) + break + case 'tool_call': + // The round's text is complete; text after the tool result is a new message. + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_call_arguments': + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { arguments: event.arguments }) + break + case 'tool_execution': + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_result': + upsertTool(event.call_id, event.function_name, { + status: event.success ? 'success' : 'error', + result: event.result, + success: event.success, + // The same text Windmill persists for the tool message. + content: event.success + ? `Used ${event.function_name} tool` + : `Error executing ${event.function_name}` + }) + break + } + } + this.#set({ messages, status: 'streaming' }) + } + + async #finishTurn(turn: Turn, result: unknown, isNew: boolean): Promise { + if (!this.#turnActive(turn)) return + if (this.#state.history === 'server') { + turn.jobIds = await this.#turnJobIds(turn) + if (!this.#turnActive(turn)) return + const reconciled = await this.#reconcileTurn(turn) + if (!this.#turnActive(turn)) return + if (reconciled) { + this.#set({ status: 'idle' }) + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + if (isNew) await this.loadConversations().catch(() => {}) + return + } + // Server history just proved unreadable: the turn completes as local history. + } + let messages = this.#state.messages + let failed = false + if (isErrorResult(result)) { + // The envelope is also a legitimate result shape; the job's own status decides. + failed = await this.#api + .getCompletedResult(turn.jobId!, turn.controller.signal) + .then((r) => r.success === false) + .catch(() => true) + if (!this.#turnActive(turn)) return + if (failed) { + messages = [...messages, assistantMessage(errorResultMessage(result), false, turn.jobId)] + } + } + if (!failed && !turn.streamedText) { + const answer = extractChatAnswer(result) + if (answer !== undefined) { + messages = [...messages, assistantMessage(answer, true, turn.jobId)] + } + } + this.#set({ messages: finalized(messages), status: 'idle' }) + this.#persistLocal() + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + } + + /** + * Folds what the server persisted for the turn into the message list. The rows + * are written by the worker in their own transactions, each of which can trail + * the flow's completion, so a streamed message whose row hasn't landed stays and + * the list is re-read a few times before the rest is kept as streamed. + * Returns false when the server holds no answer for the turn: history fell back to + * local, the read was refused or kept failing, or no assistant row has landed. The + * caller then finishes the turn from the flow result, so an answer is never lost + * to history. Whether a row counts is read from the message list, not from what + * this read returned: the turn's polling may have merged the answer already. + */ + async #reconcileTurn(turn: Turn): Promise { + const answered = () => this.#answered(turn) + for (let attempt = 1; attempt <= RECONCILE_ATTEMPTS; attempt++) { + let rows: FlowConversationMessage[] + try { + rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal: turn.controller.signal + }) + } catch (e) { + if (isAbortError(e)) throw e + if (this.#fallBackToLocal(e)) return false + const refused = e instanceof WindmillApiError && (e.status === 401 || e.status === 403) + if (refused || attempt === RECONCILE_ATTEMPTS) return answered() + await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + continue + } + if (!this.#turnActive(turn)) return true + this.#mergeRows(rows) + if (answered() && !this.#state.messages.some((m) => m.pending && m.content)) break + if (attempt < RECONCILE_ATTEMPTS) await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + } + if (!this.#turnActive(turn)) return true + this.#set({ messages: finalized(this.#state.messages) }) + return answered() + } + + /** + * The latest row the turn persisted after its user message is an assistant + * message. An agent issues each round's text row before that round's tool rows, + * and a tool row when the tool finishes, so an earlier round's text is followed + * by a tool row and only the answer closes the turn (the inserts are spawned, so + * a badly delayed one can invert that order at the cost of the reconcile + * retries). The content is not compared with the flow result: an image answer, a + * structured one and a forwarded agent result are all persisted in a shape the + * result does not reproduce. Rows carrying a job id belong to the turn when the + * job is one of the turn's, which leaves out an earlier turn whose job outlived + * `stop()` (a token without `jobs:write` cannot cancel it); a tool row without one + * (an MCP call runs inside the agent step) belongs to whatever turn is under way. + */ + #answered(turn: Turn): boolean { + const messages = this.#state.messages + const from = messages.findIndex((m) => m.id === turn.userMessageId) + const ownJob = (m: ChatMessage) => + turn.jobIds === undefined || (m.jobId === undefined ? m.role === 'tool' : turn.jobIds.has(m.jobId)) + let latest: ChatMessage | undefined + for (let i = from + 1; i < messages.length; i++) { + const m = messages[i] + if (m.seq === undefined || m.role === 'user' || !ownJob(m)) continue + if (latest === undefined || m.seq > latest.seq!) latest = m + } + return latest?.role === 'assistant' + } + + /** + * The flow job plus every step job it ran, the failure and preprocessor steps + * included (a failure handler's answer is persisted under its own job), and the + * jobs an agent step's tool calls ran as (a tool row is persisted under its own + * job too). Unknown when the read fails. + */ + async #turnJobIds(turn: Turn): Promise | undefined> { + try { + const job = await this.#api.getFlowJob(turn.jobId!, turn.controller.signal) + const ids = new Set([turn.jobId!]) + const status = job.flow_status + for (const m of [...(status?.modules ?? []), status?.failure_module, status?.preprocessor_module]) { + if (m?.job) ids.add(m.job) + for (const j of m?.flow_jobs ?? []) ids.add(j) + for (const a of m?.agent_actions ?? []) if (a.job_id) ids.add(a.job_id) + } + return ids + } catch (e) { + if (isAbortError(e)) throw e + return undefined + } + } + + #failTurn(turn: Turn, e: unknown): void { + if (!this.#turnActive(turn)) return + const error = toError(e) + this.#set({ + messages: [...finalized(this.#state.messages), assistantMessage(error.message, false, turn.jobId)], + status: 'error', + error + }) + this.#persistLocal() + this.#config.onError?.(error, { conversationId: turn.conversationId, jobId: turn.jobId }) + } + + /** Before the answer streams, earlier steps may already have persisted messages. */ + #startPolling(turn: Turn): () => void { + let stopped = false + const { signal } = turn.controller + const loop = async () => { + while (!stopped) { + try { + await sleep(POLL_INTERVAL_MS, signal) + } catch { + return + } + if (stopped) return + try { + const rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal + }) + if (!stopped && this.#turnActive(turn)) this.#mergeRows(rows) + } catch { + // transient; the completion reconciliation catches up + } + } + } + void loop() + return () => { + stopped = true + } + } + + async #syncFromServer(conversationId: string): Promise { + const rows = await this.#api.listMessages(conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100 + }) + if (this.#turn || this.#state.conversationId !== conversationId) return + this.#mergeRows(rows) + this.#set({ messages: finalized(this.#state.messages) }) + } + + /** + * Folds persisted rows into the message list. A row standing for a message the + * client already shows (same role and text; for a tool, the same tool name, since + * the server words a failure differently) takes its place under the client's id + * and keeps what only the stream knew: reasoning, call id, arguments, result. + * Other rows append in server order. Nothing is dropped: a streamed message + * outlives a row that never lands. + */ + #mergeRows(rows: FlowConversationMessage[]): void { + if (rows.length === 0) return + const messages = [...this.#state.messages] + const known = new Set(messages.map((m) => m.serverId ?? m.id)) + for (const row of rows.map(fromRow)) { + if (known.has(row.id)) continue + known.add(row.id) + const i = messages.findIndex( + (m) => + m.seq === undefined && + m.role === row.role && + (m.content === row.content || (row.tool !== undefined && m.tool?.name === row.tool.name)) + ) + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...row, + id: m.id, + reasoning: m.reasoning ?? row.reasoning, + tool: m.tool ? { ...m.tool, status: row.tool?.status ?? m.tool.status } : row.tool + } + } else { + messages.push(row) + } + } + this.#set({ messages }) + } + + #lastSeq(): number | undefined { + let last: number | undefined + for (const m of this.#state.messages) { + if (m.seq !== undefined && (last === undefined || m.seq > last)) last = m.seq + } + return last + } + + /** Writes the current messages to local history; the conversation entry itself is `#rememberConversation`'s. */ + #persistLocal(): void { + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + this.#local.saveMessages(id, this.#state.messages) + } + + /** + * Puts the current conversation at the head of local history. Only a turn moves + * a conversation there: merely viewing one must not reorder the list. + */ + #rememberConversation(): void { + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + const conversation = this.#state.conversations.find((c) => c.id === id) + if (conversation) this.#local.upsertConversation(conversation) + } + + /** Whether an unreadable server history should silently become local history. */ + #fallBackToLocal(e: unknown): boolean { + if (this.#state.history !== 'server' || this.#config.historyExplicit) return false + if (e instanceof WindmillApiError && (e.status === 401 || e.status === 403)) { + this.#set({ history: 'local' }) + // The conversation now lives in the browser; list it there like one started local. + this.#rememberConversation() + return true + } + return false + } + + #turnActive(turn: Turn): boolean { + return this.#turn === turn && this.#state.conversationId === turn.conversationId + } + + /** Stops following the current answer; the flow itself keeps running. */ + #detachTurn(): void { + const turn = this.#turn + if (!turn) return + this.#turn = undefined + turn.controller.abort() + } + + /** + * Leaves the current conversation (for another one, or because the page goes + * away). A turn still in flight is detached and what it showed so far is kept, + * written out now rather than on the debounce that may never fire. + */ + #leaveConversation(): void { + if (this.#turn) { + this.#detachTurn() + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + } + if (this.#persistTimer) this.#persistLocal() + } + + #set(patch: Partial): void { + this.#state = { ...this.#state, ...patch } + for (const listener of this.#listeners) listener(this.#state) + if (this.#state.history === 'local' && this.#state.conversationId) { + clearTimeout(this.#persistTimer) + this.#persistTimer = setTimeout(() => this.#persistLocal(), PERSIST_DEBOUNCE_MS) + } + } +} + +function fromRow(row: FlowConversationMessage): ChatMessage { + const toolName = + row.message_type === 'tool' + ? /^Used (.+) tool$/.exec(row.content)?.[1] ?? /^Error executing (.+)$/.exec(row.content)?.[1] + : undefined + const success = row.success ?? true + return { + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + tool: toolName ? { name: toolName, status: success ? 'success' : 'error' } : undefined + } +} + +function fromConversation(row: FlowConversation): Conversation { + return { + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + } +} + +function assistantMessage(content: string, success: boolean, jobId: string | undefined): ChatMessage { + return { + id: `local-${randomId()}`, + role: 'assistant', + content, + success, + createdAt: now(), + jobId, + pending: false + } +} + +function finalized(messages: ChatMessage[]): ChatMessage[] { + return messages.some((m) => m.pending) + ? messages.map((m) => (m.pending ? { ...m, pending: false } : m)) + : messages +} + +function toError(e: unknown): Error { + return e instanceof Error ? e : new Error(String(e)) +} diff --git a/chat-sdk/src/config.ts b/chat-sdk/src/config.ts new file mode 100644 index 0000000000..1903149d58 --- /dev/null +++ b/chat-sdk/src/config.ts @@ -0,0 +1,85 @@ +import type { ChatOptions, FetchLike, HistoryMode, StorageLike, TokenSource } from './types' + +export interface ResolvedConfig { + flowPath: string + baseUrl: string + workspace: string + token: TokenSource | undefined + history: HistoryMode + /** The caller chose `history`; a failing server history is then an error, not a fallback. */ + historyExplicit: boolean + inputs: Record + fetch: FetchLike | undefined + storage: StorageLike | undefined + storageKey: string | undefined + pageSize: number + pollDelayMs: number | undefined + run: ChatOptions['run'] + onFinish: ChatOptions['onFinish'] + onError: ChatOptions['onError'] +} + +export interface RawAppContext { + baseUrl: string + workspace: string + /** The viewer's SDK token in a sandboxed raw app; the session cookie otherwise. */ + token?: string +} + +/** + * The Windmill a raw app bundle runs in. A sandboxed app gets `window.process.env` + * from its wrapper once the viewer consented to the declared SDK scopes; an + * unsandboxed one runs on the Windmill origin with the viewer's session and only + * has `window.ctx`. + */ +export function detectRawApp(): RawAppContext | undefined { + const g = globalThis as { + process?: { env?: Record } + ctx?: { workspace?: unknown } + location?: { origin?: string } + } + const env = g.process?.env + if (env?.WM_RAW_APP === 'true' && env.WM_TOKEN && env.BASE_URL && env.WM_WORKSPACE) { + return { baseUrl: env.BASE_URL, workspace: env.WM_WORKSPACE, token: env.WM_TOKEN } + } + const workspace = g.ctx?.workspace + const origin = g.location?.origin + if (typeof workspace === 'string' && workspace && origin && origin !== 'null') { + return { baseUrl: origin, workspace } + } + return undefined +} + +export function resolveConfig(options: ChatOptions): ResolvedConfig { + if (!options.flowPath) throw new Error('windmill-chat: flowPath is required') + const explicitToken = options.token !== undefined + const detected = + options.baseUrl && options.workspace && explicitToken ? undefined : detectRawApp() + const baseUrl = options.baseUrl ?? detected?.baseUrl + const workspace = options.workspace ?? detected?.workspace + if (!baseUrl || !workspace) { + throw new Error( + 'windmill-chat: pass baseUrl and workspace. They are only detected inside a raw app: an unsandboxed one on the Windmill origin, or a sandboxed one whose policy declares frontend SDK scopes.' + ) + } + // The raw app's token belongs to its own Windmill; it never travels to another origin. + const token = + options.token ?? (options.baseUrl === undefined || options.baseUrl === detected?.baseUrl ? detected?.token : undefined) + return { + flowPath: options.flowPath, + baseUrl, + workspace, + token, + history: options.history ?? (explicitToken ? 'local' : 'server'), + historyExplicit: options.history !== undefined, + inputs: options.inputs ?? {}, + fetch: options.fetch, + storage: options.storage, + storageKey: options.storageKey, + pageSize: options.pageSize ?? 50, + pollDelayMs: options.pollDelayMs, + run: options.run, + onFinish: options.onFinish, + onError: options.onError + } +} diff --git a/chat-sdk/src/follow.ts b/chat-sdk/src/follow.ts new file mode 100644 index 0000000000..50f101393e --- /dev/null +++ b/chat-sdk/src/follow.ts @@ -0,0 +1,71 @@ +import type { WindmillChatApi } from './api' +import { createStreamEventParser, type AgentStreamEvent } from './stream' +import { abortError, sleep } from './utils' + +const RECONNECT_DELAY_MS = 300 + +export type FollowEvent = + /** Agent events decoded from the job's result stream; empty when a chunk ended mid-line. */ + | { type: 'stream'; events: AgentStreamEvent[] } + | { type: 'completed'; result: unknown } + +/** + * Follows a job to completion across the server's stream timeouts: every + * connection resumes from the last `stream_offset`, so no delta is repeated and + * the flow is never re-run. `onOffset` reports each offset, and its loss, so a + * caller can resume later from another connection (see the AI SDK transport). + * + * The offset indexes the stream of one sub-job (`flow_stream_job_id`, the flow's + * streaming step). A retried step gets a new one, so when the id changes the + * offset is dropped and the connection reopened from that sub-job's start. + */ +export async function* followJob( + api: WindmillChatApi, + jobId: string, + options: { signal?: AbortSignal; streamOffset?: number; onOffset?: (offset: number | undefined) => void } = {} +): AsyncGenerator { + let parser = createStreamEventParser() + let offset = options.streamOffset + let streamJobId: string | undefined + while (true) { + let reopen = false + for await (const update of api.streamJob(jobId, { streamOffset: offset, signal: options.signal })) { + if (update.type === 'ping') continue + if (update.type === 'timeout') { + reopen = true + break + } + if (update.type === 'error') throw new Error(update.error) + if (update.type === 'notfound') throw new Error(`Job ${jobId} not found`) + if (update.flow_stream_job_id && update.flow_stream_job_id !== streamJobId) { + const switched = streamJobId !== undefined && offset !== undefined + streamJobId = update.flow_stream_job_id + if (switched) { + // This connection skipped the new sub-job's first chunks: start it over. + offset = undefined + options.onOffset?.(undefined) + parser = createStreamEventParser() + reopen = true + break + } + } + if (update.stream_offset !== undefined) { + offset = update.stream_offset + options.onOffset?.(offset) + } + if (update.new_result_stream) { + yield { type: 'stream', events: parser.push(update.new_result_stream) } + } + if (update.completed) { + const rest = parser.flush() + if (rest.length > 0) yield { type: 'stream', events: rest } + yield { type: 'completed', result: update.only_result } + return + } + } + if (options.signal?.aborted) throw abortError() + // The server closes the connection after its timeout; a dropped connection looks + // the same minus the event. Either way the offset lets the next one resume. + if (!reopen) await sleep(RECONNECT_DELAY_MS, options.signal) + } +} diff --git a/chat-sdk/src/history.ts b/chat-sdk/src/history.ts new file mode 100644 index 0000000000..485dd056cf --- /dev/null +++ b/chat-sdk/src/history.ts @@ -0,0 +1,90 @@ +import type { ChatMessage, Conversation, StorageLike } from './types' + +export interface LocalHistory { + listConversations(): Conversation[] + getMessages(conversationId: string): ChatMessage[] + upsertConversation(conversation: Conversation): void + saveMessages(conversationId: string, messages: ChatMessage[]): void + deleteConversation(conversationId: string): void +} + +interface Snapshot { + v: 1 + conversations: Conversation[] + messages: Record +} + +const MAX_CONVERSATIONS = 100 + +/** + * Conversation history in the browser, for a credential shared by every visitor + * (server history would show them each other's chats). One key per flow, and per + * `storageKey` when the caller sets one; every operation re-reads the store so + * several tabs stay consistent. + */ +export function createLocalHistory(storage: StorageLike | undefined, key: string): LocalHistory { + const store = storage ?? defaultStorage() + const read = (): Snapshot => { + try { + const raw = store.getItem(key) + if (raw) { + const parsed = JSON.parse(raw) as Snapshot + if (parsed && parsed.v === 1) return parsed + } + } catch { + // unreadable: start over + } + return { v: 1, conversations: [], messages: {} } + } + const write = (snapshot: Snapshot) => { + try { + store.setItem(key, JSON.stringify(snapshot)) + } catch { + // quota or private mode: the page keeps working from memory + } + } + return { + listConversations: () => read().conversations, + getMessages: (id) => read().messages[id] ?? [], + upsertConversation(conversation) { + const s = read() + const rest = s.conversations.filter((c) => c.id !== conversation.id) + s.conversations = [conversation, ...rest] + for (const dropped of s.conversations.splice(MAX_CONVERSATIONS)) { + delete s.messages[dropped.id] + } + write(s) + }, + saveMessages(id, messages) { + const s = read() + s.messages[id] = messages.map((m) => ({ ...m, pending: false })) + write(s) + }, + deleteConversation(id) { + const s = read() + s.conversations = s.conversations.filter((c) => c.id !== id) + delete s.messages[id] + write(s) + } + } +} + +function defaultStorage(): StorageLike { + try { + const ls = globalThis.localStorage + if (ls) { + const probe = '__windmill_chat_probe__' + ls.setItem(probe, '1') + ls.removeItem(probe) + return ls + } + } catch { + // no localStorage (SSR, blocked storage): fall through + } + const memory = new Map() + return { + getItem: (k) => memory.get(k) ?? null, + setItem: (k, v) => void memory.set(k, v), + removeItem: (k) => void memory.delete(k) + } +} diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts new file mode 100644 index 0000000000..353c814154 --- /dev/null +++ b/chat-sdk/src/index.ts @@ -0,0 +1,29 @@ +export { createChat } from './chat' +export { detectRawApp, type RawAppContext } from './config' +export { + WindmillChatApi, + WindmillApiError, + readServerSentEvents, + type WindmillChatApiOptions, + type FlowConversation, + type FlowConversationMessage, + type JobUpdateEvent, + type CompletedJobResult +} from './api' +export { parseStreamEvents, createStreamEventParser, type AgentStreamEvent } from './stream' +export { followJob, type FollowEvent } from './follow' +export { extractChatAnswer, conversationIdFor } from './utils' +export type { + Chat, + ChatMessage, + ChatOptions, + ChatRole, + ChatState, + ChatStatus, + Conversation, + FetchLike, + HistoryMode, + StorageLike, + TokenSource, + ToolInvocation +} from './types' diff --git a/chat-sdk/src/react.ts b/chat-sdk/src/react.ts new file mode 100644 index 0000000000..ecba159cb3 --- /dev/null +++ b/chat-sdk/src/react.ts @@ -0,0 +1,72 @@ +import { useEffect, useMemo, useRef, useSyncExternalStore } from 'react' +import { createChat } from './chat' +import type { Chat, ChatOptions, ChatState } from './types' + +export type UseWindmillChat = ChatState & + Pick< + Chat, + | 'sendMessage' + | 'stop' + | 'newConversation' + | 'selectConversation' + | 'loadConversations' + | 'deleteConversation' + | 'loadOlderMessages' + > & { chat: Chat } + +/** + * A chat on a chat-mode flow. The chat is created once per `flowPath`, `baseUrl`, + * `workspace`, `history`, `storageKey`, credential and presence of `run`, and + * destroyed on unmount. A credential change is a new user, whose chat must not + * carry the previous one's state: a different token string, or a switch between no + * token, a token string and a token function, all recreate it. A token function is + * read through a ref on every call, so a new closure per render changes what the + * next call runs and nothing else; pass a `storageKey` per user when local history + * must not be shared. `run`, the callbacks and `inputs` are read the same way: the + * latest render's values go with the next message. + */ +export function useWindmillChat(options: ChatOptions): UseWindmillChat { + const latest = useRef(options) + latest.current = options + const credential = + typeof options.token === 'function' ? 'fn' : typeof options.token === 'string' ? `str:${options.token}` : 'none' + // A custom runner replaces the deployed flow call, so its presence is part of what the chat is. + const customRun = options.run !== undefined + const chat = useMemo( + () => + createChat({ + ...options, + // Sent per message from the latest render instead, so a removed key stays removed. + inputs: undefined, + token: + typeof options.token === 'function' + ? () => { + const token = latest.current.token + return typeof token === 'function' ? token() : (token ?? '') + } + : options.token, + run: customRun ? (args, turn) => (latest.current.run ?? options.run!)(args, turn) : undefined, + onFinish: (turn) => latest.current.onFinish?.(turn), + onError: (error, turn) => latest.current.onError?.(error, turn) + }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [options.flowPath, options.baseUrl, options.workspace, options.history, options.storageKey, credential, customRun] + ) + useEffect(() => () => chat.destroy(), [chat]) + const state = useSyncExternalStore(chat.subscribe, chat.getState, chat.getState) + return useMemo( + () => ({ + ...state, + chat, + sendMessage: (text, options) => + chat.sendMessage(text, { ...options, inputs: { ...latest.current.inputs, ...options?.inputs } }), + stop: chat.stop, + newConversation: chat.newConversation, + selectConversation: chat.selectConversation, + loadConversations: chat.loadConversations, + deleteConversation: chat.deleteConversation, + loadOlderMessages: chat.loadOlderMessages + }), + [state, chat] + ) +} diff --git a/chat-sdk/src/stream.ts b/chat-sdk/src/stream.ts new file mode 100644 index 0000000000..c6969bb7b4 --- /dev/null +++ b/chat-sdk/src/stream.ts @@ -0,0 +1,65 @@ +/** The events an AI agent step streams, one JSON object per line of the job's result stream. */ +export type AgentStreamEvent = + | { type: 'token_delta'; content: string } + | { type: 'reasoning_token_delta'; content: string } + | { type: 'tool_call'; call_id: string; function_name: string } + | { type: 'tool_call_arguments'; call_id: string; function_name: string; arguments: string } + | { type: 'tool_execution'; call_id: string; function_name: string } + | { + type: 'tool_result' + call_id: string + function_name: string + result: string + success: boolean + } + +const KNOWN_TYPES = new Set([ + 'token_delta', + 'reasoning_token_delta', + 'tool_call', + 'tool_call_arguments', + 'tool_execution', + 'tool_result' +]) + +/** + * Incremental parser for the `new_result_stream` chunks of a job update. A chunk is + * not guaranteed to end on a line boundary, so an incomplete last line waits for the + * next `push` (or `flush` once the job completes). + */ +export function createStreamEventParser() { + let pending = '' + return { + push(chunk: string): AgentStreamEvent[] { + pending += chunk + const lastNewline = pending.lastIndexOf('\n') + if (lastNewline === -1) return [] + const complete = pending.slice(0, lastNewline) + pending = pending.slice(lastNewline + 1) + return parseStreamEvents(complete) + }, + flush(): AgentStreamEvent[] { + const rest = pending + pending = '' + return parseStreamEvents(rest) + } + } +} + +/** Parses complete NDJSON lines; lines that aren't agent events are skipped. */ +export function parseStreamEvents(ndjson: string): AgentStreamEvent[] { + const events: AgentStreamEvent[] = [] + for (const line of ndjson.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + try { + const parsed = JSON.parse(trimmed) + if (parsed && typeof parsed === 'object' && KNOWN_TYPES.has(parsed.type)) { + events.push(parsed as AgentStreamEvent) + } + } catch { + // not an agent event + } + } + return events +} diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts new file mode 100644 index 0000000000..92b8dfb89b --- /dev/null +++ b/chat-sdk/src/types.ts @@ -0,0 +1,136 @@ +export type ChatRole = 'user' | 'assistant' | 'tool' | 'system' + +/** + * - `idle`: ready for a message + * - `submitted`: the message was sent, no answer has started streaming yet + * - `streaming`: the answer is arriving + * - `error`: the last turn failed; `error` holds why. Sending again is allowed. + */ +export type ChatStatus = 'idle' | 'submitted' | 'streaming' | 'error' + +/** + * Where conversation history lives. + * - `server`: Windmill's conversation store. Each Windmill user only sees their own + * conversations, so use it with the viewer's own session or a per-user token. + * - `local`: the browser's storage. Right for a token shared by every visitor. + * - `none`: nothing is kept beyond the current page. + */ +export type HistoryMode = 'server' | 'local' | 'none' + +export interface ToolInvocation { + callId?: string + name: string + /** The arguments the model passed, as a JSON string. */ + arguments?: string + result?: string + status: 'running' | 'success' | 'error' +} + +export interface ChatMessage { + id: string + role: ChatRole + content: string + /** The model's reasoning summary, when the provider streams one. */ + reasoning?: string + /** Set on `tool` messages that came from the live stream. */ + tool?: ToolInvocation + success: boolean + createdAt: string + jobId?: string + /** The flow step that produced the message. */ + stepName?: string + /** True while the message is optimistic or still streaming. */ + pending: boolean + /** Id of the persisted row once the server has it; `id` itself never changes, so list keys stay stable. */ + serverId?: string + /** The server's cursor for a persisted message; unset for one created on the client. */ + seq?: number +} + +export interface Conversation { + id: string + title: string | undefined + createdAt: string + updatedAt: string +} + +export interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: ChatStatus + error: Error | undefined + conversations: Conversation[] + /** Where history is read from. Starts as configured; drops from `server` to `local` when the credential cannot read conversations. */ + history: HistoryMode + loadingMessages: boolean + hasMoreMessages: boolean +} + +export type TokenSource = string | (() => string | Promise) + +export type StorageLike = Pick + +export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise + +export interface ChatOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Windmill origin, e.g. `https://app.windmill.dev`. Detected inside a raw app. */ + baseUrl?: string + /** Detected inside a raw app. */ + workspace?: string + /** + * A Windmill token, or a function returning one (called before every request, so it + * can fetch a short-lived token from your backend). Omit it to use the viewer's + * session: the cookie on the Windmill origin, or a sandboxed raw app's SDK token. + */ + token?: TokenSource + /** Defaults to `server` with the viewer's session and `local` with an explicit token. */ + history?: HistoryMode + /** Extra flow inputs sent with every message, next to `user_message`. */ + inputs?: Record + fetch?: FetchLike + /** Backing store for `local` history. Defaults to `localStorage`. */ + storage?: StorageLike + /** + * Namespace for `local` history, e.g. the signed-in user's id. Local history is + * per browser, per flow; without this, users sharing a browser share it. + */ + storageKey?: string + /** Messages fetched per page of server history. */ + pageSize?: number + /** + * How often, in milliseconds, the server polls a running turn for the stream + * (Enterprise; 50 at the fastest, other servers ignore it). Unset, the server + * relaxes from 100 ms to 3 s over a long turn. + */ + pollDelayMs?: number + /** + * Runs the flow for a turn and returns the job id, instead of the deployed flow at + * `flowPath`. `args` carries `user_message` and the extra inputs; the run must set + * `memory_id` to the conversation id for the conversation and its memory to line up. + * Windmill's own editor uses this to chat with an undeployed flow through a preview run. + */ + run?: (args: Record, turn: { conversationId: string; signal: AbortSignal }) => Promise + /** Called once a turn has its answer (a failed flow included: its error is the answer). */ + onFinish?: (turn: { conversationId: string; jobId?: string; messages: ChatMessage[] }) => void + /** Called when a turn could not run or be followed; `state.error` holds the same error. */ + onError?: (error: Error, turn: { conversationId: string; jobId?: string }) => void +} + +export interface Chat { + getState(): ChatState + /** Calls `listener` now and on every change; returns the unsubscribe function (Svelte store contract). */ + subscribe(listener: (state: ChatState) => void): () => void + /** Sends a message in the current conversation, starting one when there is none. Resolves when the answer is complete. */ + sendMessage(text: string, options?: { inputs?: Record }): Promise + /** Stops following the answer and asks Windmill to cancel the run. */ + stop(): Promise + newConversation(): void + selectConversation(conversationId: string): Promise + loadConversations(options?: { page?: number; perPage?: number }): Promise + deleteConversation(conversationId: string): Promise + loadOlderMessages(): Promise + /** Stops background work (stream, polling) and writes local history out. The chat stays usable. */ + destroy(): void +} diff --git a/chat-sdk/src/utils.ts b/chat-sdk/src/utils.ts new file mode 100644 index 0000000000..fe7e5d8960 --- /dev/null +++ b/chat-sdk/src/utils.ts @@ -0,0 +1,134 @@ +export function randomId(): string { + const c = globalThis.crypto + if (c?.randomUUID) return c.randomUUID() + // `randomUUID` needs a secure context; a plain http dev origin has `getRandomValues` only. + const bytes = new Uint8Array(16) + c.getRandomValues(bytes) + return formatUuid(bytes, 4) +} + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +export function isUuid(value: string): boolean { + return UUID_RE.test(value) +} + +/** + * The Windmill conversation id for an arbitrary chat id. A UUID is used as is; + * anything else (an AI SDK chat id, for instance) maps to the same UUID every time, + * so a page can reopen its conversation without storing a second id. Hashed in plain + * JS: `crypto.subtle` only exists in secure contexts, and the mapping must not depend + * on the origin's scheme. + */ +export function conversationIdFor(chatId: string): string { + if (isUuid(chatId)) return chatId.toLowerCase() + const bytes = new TextEncoder().encode(`windmill-chat:${chatId}`) + const out = new Uint8Array(16) + for (const [i, seed] of [0xcbf29ce484222325n, 0x84222325cbf29ce4n].entries()) { + let h = fnv1a64(bytes, seed) + for (let b = 7; b >= 0; b--) { + out[i * 8 + b] = Number(h & 0xffn) + h >>= 8n + } + } + return formatUuid(out, 5) +} + +function fnv1a64(bytes: Uint8Array, seed: bigint): bigint { + let h = seed + for (const byte of bytes) { + h ^= BigInt(byte) + h = (h * 0x100000001b3n) & 0xffffffffffffffffn + } + return h +} + +function formatUuid(bytes: Uint8Array, version: 4 | 5): string { + bytes[6] = (bytes[6] & 0x0f) | (version << 4) + bytes[8] = (bytes[8] & 0x3f) | 0x80 + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +export function parseJsonOr(text: string | undefined): unknown { + if (text === undefined) return undefined + try { + return JSON.parse(text) + } catch { + return text + } +} + +export function now(): string { + return new Date().toISOString() +} + +/** Same rule as the server: the first message, cut to 25 characters. */ +export function conversationTitle(firstMessage: string): string { + const chars = Array.from(firstMessage) + return chars.length > 25 ? `${chars.slice(0, 25).join('')}...` : firstMessage +} + +export function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(abortError()) + const onAbort = () => { + clearTimeout(timer) + reject(abortError()) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + +export function abortError(): Error { + return new DOMException('The operation was aborted', 'AbortError') +} + +export function isAbortError(e: unknown): boolean { + return e instanceof Error && e.name === 'AbortError' +} + +/** + * The text a chat shows for a flow result, following what Windmill persists as the + * assistant message when the last step is not an AI agent: `windmill_chat_answer` + * when the result carries one (null means no message), an agent result's `output`, + * a string as is, anything else as JSON. + */ +export function extractChatAnswer(result: unknown): string | undefined { + if (result === null || result === undefined) return undefined + if (typeof result === 'string') return result + if (typeof result === 'object' && !Array.isArray(result)) { + const obj = result as Record + if ('windmill_chat_answer' in obj) return formatAnswer(obj.windmill_chat_answer) + if ('output' in obj && Array.isArray(obj.messages)) return formatAnswer(obj.output) + } + return JSON.stringify(result, null, 2) +} + +function formatAnswer(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined + return typeof value === 'string' ? value : JSON.stringify(value, null, 2) +} + +/** A completed job whose result is Windmill's error envelope. */ +export function isErrorResult(result: unknown): result is { error: Record } { + return ( + typeof result === 'object' && + result !== null && + 'error' in result && + typeof (result as { error: unknown }).error === 'object' && + (result as { error: unknown }).error !== null + ) +} + +export function errorResultMessage(result: { error: Record }): string { + const { message, name } = result.error + if (typeof message === 'string' && message) { + return typeof name === 'string' && name && name !== 'Error' ? `${name}: ${message}` : message + } + return JSON.stringify(result.error, null, 2) +} diff --git a/chat-sdk/test/ai-sdk-chat.test.ts b/chat-sdk/test/ai-sdk-chat.test.ts new file mode 100644 index 0000000000..a0bbb7b8a1 --- /dev/null +++ b/chat-sdk/test/ai-sdk-chat.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from 'bun:test' +import { Chat } from '@ai-sdk/react' +import { createWindmillChatTransport } from '../src/ai-sdk' +import { fetchMock, json, ndjson, sse, text } from './support' + +const FLOW = 'f/chat/agent' + +/** The AI SDK's own chunk processor consuming the transport, as `useChat` would. */ +describe('AI SDK Chat over the Windmill transport', () => { + test('builds the assistant message parts and settles to ready', async () => { + let jobs = 0 + const { fetch, calls } = fetchMock( + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text(`job-${++jobs}`) : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'reasoning_token_delta', content: 'so ' }, + { type: 'token_delta', content: 'The answer ' }, + { type: 'token_delta', content: 'is 42' } + ), + stream_offset: 6, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { error: { message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/completed/get_result_maybe/job-2') ? json({ completed: true, success: false }) : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, token: 'tok', fetch }) + const chat = new Chat({ id: 'e2e-chat', transport }) + + await chat.sendMessage({ text: 'what is it?' }) + + expect(chat.status).toBe('ready') + expect(chat.messages.map((m) => m.role)).toEqual(['user', 'assistant']) + const parts = chat.messages[1].parts + expect(parts.map((p) => p.type)).toEqual(['dynamic-tool', 'reasoning', 'text']) + expect(parts[0]).toMatchObject({ toolName: 'lookup', toolCallId: 'c1', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(parts[1]).toMatchObject({ type: 'reasoning', text: 'so ', state: 'done' }) + expect(parts[2]).toMatchObject({ type: 'text', text: 'The answer is 42', state: 'done' }) + // Both turns of the chat ran in the same Windmill conversation. + const memoryIds = calls.filter((c) => c.method === 'POST').map((c) => c.url.searchParams.get('memory_id')) + expect(memoryIds[0]).toBe(transport.conversationId('e2e-chat')) + + await chat.sendMessage({ text: 'and now fail' }) + expect(chat.status).toBe('error') + expect(chat.error?.message).toBe('boom') + expect(memoryIds.length === 1 || calls.filter((c) => c.method === 'POST')[1].url.searchParams.get('memory_id') === memoryIds[0]).toBe(true) + }) +}) diff --git a/chat-sdk/test/ai-sdk.test.ts b/chat-sdk/test/ai-sdk.test.ts new file mode 100644 index 0000000000..7f30f26edc --- /dev/null +++ b/chat-sdk/test/ai-sdk.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from 'bun:test' +import type { UIMessage, UIMessageChunk } from 'ai' +import { createWindmillChatTransport, toUIMessages } from '../src/ai-sdk' +import type { ChatMessage } from '../src/types' +import { fetchMock, json, ndjson, sse, text, type Route } from './support' + +const FLOW = 'f/chat/agent' +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +const userMessage = (text: string): UIMessage => ({ id: 'u1', role: 'user', parts: [{ type: 'text', text }] }) + +async function collect(stream: ReadableStream): Promise { + const chunks: UIMessageChunk[] = [] + const reader = stream.getReader() + while (true) { + const { value, done } = await reader.read() + if (done) return chunks + chunks.push(value) + } +} + +describe('createWindmillChatTransport', () => { + test('maps an agent turn to AI SDK chunks and derives the conversation from the chat id', async () => { + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'think' }, + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_execution', call_id: 'c1', function_name: 'lookup' } + ), + stream_offset: 5 + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '{"answer":42}', success: true }, + { type: 'token_delta', content: '42' } + ), + stream_offset: 7, + completed: true, + only_result: { output: '42', messages: [] } + } + ]) + : undefined + ) + const transport = createWindmillChatTransport({ + baseUrl: 'http://wm.test', + workspace: 'ws', + flowPath: FLOW, + token: 'tok', + inputs: { tone: 'kind' }, + fetch + }) + const chunks = await collect( + await transport.sendMessages({ + trigger: 'submit-message', + chatId: 'chat-abc', + messageId: undefined, + messages: [userMessage('what is it?')], + abortSignal: undefined, + body: { locale: 'fr' } + }) + ) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.body).toEqual({ tone: 'kind', locale: 'fr', user_message: 'what is it?' }) + expect(runCall.url.searchParams.get('memory_id')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abc')).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) + expect(transport.conversationId('chat-abc')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abd')).not.toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('4D6E5C8C-2C3B-4E1A-9F31-0B2E6F1C9D10')).toBe( + '4d6e5c8c-2c3b-4e1a-9f31-0b2e6f1c9d10' + ) + + const shape = chunks.map((c) => ('delta' in c ? `${c.type}:${c.delta}` : c.type)) + expect(shape).toEqual([ + 'start', + 'reasoning-start', + 'reasoning-delta:think', + 'text-start', + 'text-delta:Let me ', + 'reasoning-end', + 'text-end', + 'tool-input-start', + 'tool-input-available', + 'tool-output-available', + 'text-start', + 'text-delta:42', + 'text-end', + 'finish' + ]) + expect(chunks.find((c) => c.type === 'tool-input-available')).toMatchObject({ + toolCallId: 'c1', + toolName: 'lookup', + input: { q: 1 }, + dynamic: true + }) + expect(chunks.find((c) => c.type === 'tool-output-available')).toMatchObject({ output: { answer: 42 } }) + // The job finished, so there is nothing to reconnect to. + expect(await transport.reconnectToStream({ chatId: 'chat-abc' })).toBeNull() + }) + + test('answers from the flow result when nothing streamed, and reports a failed flow as an error', async () => { + let turn = 0 + const { fetch } = fetchMock( + run, + (c) => { + if (c.url.pathname !== streamPath) return undefined + turn++ + return sse([ + turn === 1 + ? { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } } + : { type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } } + ]) + }, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false }) + : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch }) + const send = () => + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [userMessage('hi')], + abortSignal: undefined + }) + const first = await collect(await send()) + expect(first.map((c) => ('delta' in c ? c.delta : c.type))).toEqual(['start', 'text-start', 'From a script', 'text-end', 'finish']) + const second = await collect(await send()) + expect(second.map((c) => c.type)).toEqual(['start', 'error']) + expect(second[1]).toMatchObject({ errorText: 'ExecutionErr: boom' }) + }) + + test('refuses attachments with a clear error', async () => { + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch: fetchMock().fetch }) + await expect( + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [{ id: 'u', role: 'user', parts: [{ type: 'file', mediaType: 'image/png', url: 'data:...' }] }], + abortSignal: undefined + }) + ).rejects.toThrow('attachments are not supported') + }) +}) + +describe('toUIMessages', () => { + test('folds a turn into one assistant message with reasoning, tool and text parts', () => { + const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'thanks' }, + { ...base, id: 't2', role: 'tool', content: 'Error executing lookup', success: false, tool: { name: 'lookup', status: 'error' } } + ] + const ui = toUIMessages(messages) + expect(ui.map((m) => [m.id, m.role, m.parts.map((p) => p.type)])).toEqual([ + ['u1', 'user', ['text']], + ['t1', 'assistant', ['dynamic-tool', 'reasoning', 'text']], + ['u2', 'user', ['text']], + ['t2', 'assistant', ['dynamic-tool']] + ]) + expect(ui[1].parts[0]).toMatchObject({ toolCallId: 'c1', toolName: 'lookup', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(ui[3].parts[0]).toMatchObject({ state: 'output-error', errorText: 'Error executing lookup' }) + }) +}) diff --git a/chat-sdk/test/assistant-ui.test.ts b/chat-sdk/test/assistant-ui.test.ts new file mode 100644 index 0000000000..5641dc2907 --- /dev/null +++ b/chat-sdk/test/assistant-ui.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from 'bun:test' +import { groupTurns, toThreadMessage } from '../src/assistant-ui' +import type { ChatMessage } from '../src/types' + +const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + +describe('assistant-ui conversion', () => { + test('groups rows into turns and renders tool calls as content parts', () => { + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'again' }, + { ...base, id: 'a2', role: 'assistant', content: 'partial', pending: true } + ] + const turns = groupTurns(messages) + expect(turns.map((t) => [t.id, t.role, t.messages.length])).toEqual([ + ['u1', 'user', 1], + ['t1', 'assistant', 2], + ['u2', 'user', 1], + ['a2', 'assistant', 1] + ]) + const answer = toThreadMessage(turns[1]) + expect(answer).toMatchObject({ id: 't1', role: 'assistant', status: { type: 'complete', reason: 'stop' } }) + expect(answer.content).toEqual([ + { type: 'tool-call', toolCallId: 'c1', toolName: 'lookup', args: { q: 1 }, argsText: '{"q":1}', result: 42, isError: false }, + { type: 'reasoning', text: 'hmm' }, + { type: 'text', text: 'The answer is 42' } + ]) + expect(toThreadMessage(turns[3]).status).toEqual({ type: 'running' }) + expect(toThreadMessage(turns[0])).toMatchObject({ role: 'user', content: [{ type: 'text', text: 'hi' }] }) + }) + + test('marks a failed answer as incomplete', () => { + const [turn] = groupTurns([{ ...base, id: 'a', role: 'assistant', content: 'boom', success: false }]) + expect(toThreadMessage(turn).status).toEqual({ type: 'incomplete', reason: 'error', error: 'boom' }) + }) +}) diff --git a/chat-sdk/test/chat.test.ts b/chat-sdk/test/chat.test.ts new file mode 100644 index 0000000000..1a5be8375d --- /dev/null +++ b/chat-sdk/test/chat.test.ts @@ -0,0 +1,822 @@ +import { describe, expect, test } from 'bun:test' +import { createChat } from '../src/chat' +import type { ChatOptions } from '../src/types' +import { fetchMock, json, memoryStorage, messageRow, ndjson, sse, sseTimed, text, type Route } from './support' + +const BASE = 'http://wm.test' +const FLOW = 'f/chat/agent' + +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined + +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +function options(extra: Partial, fetch: ChatOptions['fetch']): ChatOptions { + return { flowPath: FLOW, baseUrl: BASE, workspace: 'ws', fetch, storage: memoryStorage(), ...extra } +} + +describe('createChat with local history', () => { + test('streams text and tool calls, then finalizes and persists the turn', async () => { + const storage = memoryStorage() + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { type: 'ping' }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' } + ), + stream_offset: 3, + flow_stream_job_id: 'agent-job' + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'token_delta', content: 'The answer is 42' } + ), + stream_offset: 5, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const statuses: string[] = [] + chat.subscribe((s) => statuses.push(s.status)) + + await chat.sendMessage('what is the answer?', { inputs: { locale: 'fr' } }) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.url.searchParams.get('memory_id')).toBe(chat.getState().conversationId!) + expect(runCall.body).toEqual({ locale: 'fr', user_message: 'what is the answer?' }) + expect(runCall.headers.authorization).toBe('Bearer tok') + + const state = chat.getState() + expect(state.status).toBe('idle') + expect(statuses).toContain('submitted') + expect(statuses).toContain('streaming') + expect(state.messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'what is the answer?', false], + ['assistant', 'Let me ', false], + ['tool', 'Used lookup tool', false], + ['assistant', 'The answer is 42', false] + ]) + expect(state.messages[2].tool).toEqual({ + callId: 'c1', + name: 'lookup', + status: 'success', + arguments: '{"q":1}', + result: '42' + }) + expect(state.conversations).toHaveLength(1) + expect(state.conversations[0].title).toBe('what is the answer?') + + const reloaded = createChat(options({ token: 'tok', storage }, fetch)) + await reloaded.loadConversations() + expect(reloaded.getState().conversations.map((c) => c.id)).toEqual([state.conversationId!]) + await reloaded.selectConversation(state.conversationId!) + expect(reloaded.getState().messages.map((m) => m.content)).toEqual( + state.messages.map((m) => m.content) + ) + }) + + test('a tool call id reused by a later turn gets its own message', async () => { + const toolTurn = () => + sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'same-id', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'same-id', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'done' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'done', messages: [] } + } + ]) + const { fetch } = fetchMock(run, (c) => (c.url.pathname === streamPath ? toolTurn() : undefined)) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('one') + await chat.sendMessage('two') + expect(chat.getState().messages.map((m) => m.role)).toEqual([ + 'user', + 'tool', + 'assistant', + 'user', + 'tool', + 'assistant' + ]) + }) + + test('a custom run replaces the deployed flow call and still follows the job', async () => { + const { fetch, calls } = fetchMock((c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'from preview' } }]) + : undefined + ) + const seen: unknown[] = [] + const chat = createChat( + options( + { + token: 'tok', + inputs: { tone: 'kind' }, + run: async (args, turn) => { + seen.push({ args, conversationId: turn.conversationId, aborted: turn.signal.aborted }) + return 'job-1' + } + }, + fetch + ) + ) + await chat.sendMessage('hi') + expect(seen).toEqual([{ args: { tone: 'kind', user_message: 'hi' }, conversationId: chat.getState().conversationId, aborted: false }]) + expect(calls.filter((c) => c.method === 'POST')).toHaveLength(0) + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'from preview']) + }) + + test('re-attaches from the start when the streaming step is retried under a new sub-job', async () => { + let streams = 0 + const { fetch, calls } = fetchMock(run, (c) => { + if (c.url.pathname !== streamPath) return undefined + streams++ + if (streams === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'first try ' }), stream_offset: 1, flow_stream_job_id: 'agent-1' }, + // The retried step streams under a new sub-job; the offset above indexes the old one. + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'y ' }), stream_offset: 2, flow_stream_job_id: 'agent-2' } + ]) + } + return sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'second try' }), + stream_offset: 1, + flow_stream_job_id: 'agent-2', + completed: true, + only_result: 'second try' + } + ]) + }) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const streamCalls = calls.filter((c) => c.url.pathname === streamPath) + expect(streamCalls).toHaveLength(2) + expect(streamCalls[1].url.searchParams.get('stream_offset')).toBeNull() + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'first try second try']) + }) + + test('resumes after a stream timeout from the last offset without re-running the flow', async () => { + let streamCalls = 0 + const { fetch, calls } = fetchMock(run, (c) => { + if (c.url.pathname !== streamPath) return undefined + streamCalls++ + if (streamCalls === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'Hel' }), stream_offset: 1 }, + { type: 'timeout' } + ]) + } + return sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'lo' }), + stream_offset: 2, + completed: true, + only_result: 'Hello' + } + ]) + }) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + + expect(calls.filter((c) => c.method === 'POST')).toHaveLength(1) + const streams = calls.filter((c) => c.url.pathname === streamPath) + expect(streams).toHaveLength(2) + expect(streams[0].url.searchParams.get('stream_offset')).toBeNull() + expect(streams[1].url.searchParams.get('stream_offset')).toBe('1') + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'Hello']) + }) + + test('derives the answer from the flow result when nothing streamed', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const [, answer] = chat.getState().messages + expect(answer.role).toBe('assistant') + expect(answer.content).toBe('From a script') + expect(answer.jobId).toBe('job-1') + }) + + test('renders a successful result that merely looks like an error envelope', async () => { + const result = { error: { message: 'domain data' } } + const { fetch } = fetchMock( + run, + (c) => (c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: result }]) : undefined), + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: true, result }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages[1]).toMatchObject({ + role: 'assistant', + success: true, + content: JSON.stringify(result, null, 2) + }) + }) + + test('reports a failed flow as an unsuccessful assistant message', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + completed: true, + only_result: { error: { name: 'ExecutionErr', message: 'boom' } } + } + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false, result: { error: { message: 'boom' } } }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.status).toBe('idle') + expect(state.messages[1]).toMatchObject({ role: 'assistant', success: false, content: 'ExecutionErr: boom' }) + }) +}) + +describe('createChat with server history', () => { + test('replaces the optimistic turn with the persisted rows', async () => { + const { fetch, calls } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'hmm' }, + { type: 'token_delta', content: 'Hello' } + ), + stream_offset: 2, + completed: true, + only_result: { output: 'Hello', messages: [] } + } + ]) + : undefined, + (c) => + c.method === 'GET' && c.url.pathname.endsWith('/messages') + ? json([ + messageRow(11, 'user', 'hi'), + messageRow(12, 'assistant', 'Hello', { step_name: 'AI Agent', job_id: 'agent-job' }) + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json([ + { + id: chatId, + workspace_id: 'ws', + flow_path: FLOW, + title: 'hi', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:01Z', + created_by: 'admin' + } + ]) + : undefined + ) + const chat = createChat(options({}, fetch)) + let chatId = '' + const unsubscribe = chat.subscribe((s) => { + chatId = s.conversationId ?? chatId + }) + await chat.sendMessage('hi') + unsubscribe() + + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-11', 'user', 'hi', false], + ['row-12', 'assistant', 'Hello', false] + ]) + // Ids stay the client's, so list keys never remount; the server id rides alongside. + expect(state.messages.map((m) => m.id.startsWith('pending-'))).toEqual([true, true]) + expect(state.messages[1]).toMatchObject({ reasoning: 'hmm', stepName: 'AI Agent', jobId: 'agent-job', seq: 12 }) + expect(state.conversations.map((c) => c.id)).toEqual([chatId]) + + const messagesCall = calls.find((c) => c.url.pathname.endsWith('/messages'))! + expect(messagesCall.url.pathname).toBe(`/api/w/ws/flow_conversations/${chatId}/messages`) + expect(messagesCall.headers.authorization).toBeUndefined() + }) + + test('keeps the streamed answer until its row lands, even when a tool row lands first', async () => { + let messageFetches = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'Final answer' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'Final answer', messages: [] } + } + ]) + : undefined, + (c) => { + if (!c.url.pathname.endsWith('/messages')) return undefined + messageFetches++ + // The assistant row is written by a task that trails the tool's. + return json( + messageFetches === 1 + ? [messageRow(21, 'user', 'hi'), messageRow(22, 'tool', 'Used lookup tool')] + : [messageRow(23, 'assistant', 'Final answer')] + ) + }, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(messageFetches).toBe(2) + expect(chat.getState().messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-21', 'user', 'hi', false], + ['row-22', 'tool', 'Used lookup tool', false], + ['row-23', 'assistant', 'Final answer', false] + ]) + expect(chat.getState().messages[1].tool).toMatchObject({ callId: 'c1', result: '1', status: 'success' }) + }) + + test('finishes the turn from the flow result when history falls back mid-turn', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('forbidden', 403) : undefined) + ) + const chat = createChat(options({ storage }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('local') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + const stored = JSON.parse([...storage.data.values()][0]) + expect(stored.messages[state.conversationId!]).toHaveLength(2) + }) + + test('appends later pages of conversations', async () => { + const row = (id: string) => ({ + id, + workspace_id: 'ws', + flow_path: FLOW, + title: id, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + created_by: 'admin' + }) + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json(c.url.searchParams.get('page') === '2' ? [row('c2')] : [row('c1')]) + : undefined + ) + const chat = createChat(options({}, fetch)) + await chat.loadConversations() + await chat.loadConversations({ page: 2 }) + expect(chat.getState().conversations.map((c) => c.id)).toEqual(['c1', 'c2']) + }) + + test('a turn started right after stop() is not touched by the stop sync', async () => { + let jobs = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'slow...' }), stream_offset: 1 }]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c2', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c2', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'second' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'second', messages: [] } + } + ]) + : undefined, + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('ok') : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(31, 'user', 'first'), messageRow(32, 'user', 'second question'), messageRow(33, 'tool', 'Used lookup tool'), messageRow(34, 'assistant', 'second')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + // The stream of job-1 never completes: the connection just ends, so the turn keeps waiting. + await new Promise((r) => setTimeout(r, 50)) + const stopped = chat.stop() + await first + const second = chat.sendMessage('second question') + await stopped + await second + const roles = chat.getState().messages.map((m) => `${m.role}${m.pending ? '*' : ''}`) + expect(roles).toEqual(['user', 'assistant', 'user', 'tool', 'assistant']) + expect(chat.getState().messages.filter((m) => m.role === 'tool')).toHaveLength(1) + }) + + test('answers from the flow result when only the user row has been persisted', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/messages') ? (reads++, json([messageRow(41, 'user', 'hi')])) : undefined), + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBeGreaterThan(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-41'], + ['assistant', 'From a script', undefined] + ]) + }) + + test('an answer the poller merged before completion is not appended again', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sseTimed([{ type: 'update' }, 1400, { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(51, 'user', 'hi'), messageRow(52, 'assistant', 'From a script')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-51'], + ['assistant', 'From a script', 'row-52'] + ]) + }) + + test('an earlier round of a non-streaming agent is not its answer', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Final answer', messages: [] } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-1') + ? json({ flow_status: { modules: [{ job: 'step-1', agent_actions: [{ type: 'tool_call', job_id: 'tool-1' }, { type: 'message' }] }] } }) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(91, 'user', 'hi'), messageRow(92, 'assistant', 'Let me check', { job_id: 'step-1' }), messageRow(93, 'tool', 'Used lookup tool', { job_id: 'tool-1' })] + : [messageRow(94, 'assistant', 'Final answer', { job_id: 'step-1' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-91'], + ['assistant', 'Let me check', 'row-92'], + ['tool', 'Used lookup tool', 'row-93'], + ['assistant', 'Final answer', 'row-94'] + ]) + }) + + test('an answer persisted in another shape than the flow result is still the answer', async () => { + // An image agent returns the S3 object and persists it with a type marker. + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { s3: 'agent/img.png' } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/jobs_u/get/job-1') ? json({ flow_status: { modules: [{ job: 'step-1' }] } }) : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json(++reads === 1 ? [messageRow(71, 'user', 'draw'), messageRow(72, 'assistant', '{"s3":"agent/img.png","type":"windmill_s3_object"}', { job_id: 'step-1' })] : []) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('draw') + expect(reads).toBe(1) + expect(chat.getState().messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'draw'], + ['assistant', '{"s3":"agent/img.png","type":"windmill_s3_object"}'] + ]) + }) + + test('a tool row without a job (an MCP call) still separates a round from the answer', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Final answer', messages: [] } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/jobs_u/get/job-1') ? json({ flow_status: { modules: [{ job: 'step-1', agent_actions: [{ type: 'mcp_tool_call' }, { type: 'message' }] }] } }) : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(81, 'user', 'hi'), messageRow(82, 'assistant', 'Let me check', { job_id: 'step-1' }), messageRow(83, 'tool', 'Used search tool', { job_id: null })] + : [messageRow(84, 'assistant', 'Final answer', { job_id: 'step-1' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'Let me check', 'Used search tool', 'Final answer']) + }) + + test('the stream asks for a server poll interval only when one is set', async () => { + const answer: Route = (c) => + c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: 'ok' }]) : undefined + const plain = fetchMock(run, answer) + await createChat(options({ token: 'tok' }, plain.fetch)).sendMessage('hi') + expect(plain.calls.find((c) => c.url.pathname === streamPath)!.url.searchParams.has('poll_delay_ms')).toBe(false) + const fast = fetchMock(run, answer) + await createChat(options({ token: 'tok', pollDelayMs: 50 }, fast.fetch)).sendMessage('hi') + expect(fast.calls.find((c) => c.url.pathname === streamPath)!.url.searchParams.get('poll_delay_ms')).toBe('50') + }) + + test('a tool row alone is not the answer of a turn that streamed no text', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Answer', messages: [] } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json(++reads === 1 ? [messageRow(61, 'user', 'hi'), messageRow(62, 'tool', 'Used lookup tool')] : [messageRow(63, 'assistant', 'Answer')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-61'], + ['tool', 'Used lookup tool', 'row-62'], + ['assistant', 'Answer', 'row-63'] + ]) + }) + + test('a late answer from a stopped job is not taken as the next turn answer', async () => { + let jobs = 0 + let reads = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + // job-1 never completes: the connection just ends, so the turn keeps waiting. + (c) => (c.url.pathname.endsWith('/getupdate_sse/job-1') ? sse([{ type: 'update' }]) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'second answer' } }]) + : undefined, + // The run-only token cannot cancel: job-1 keeps running after stop(). + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('forbidden', 400) : undefined), + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-2') + ? json({ flow_status: { modules: [{ job: 'step-2' }] } }) + : undefined, + // Read 1 is stop()'s sync; the stopped job's answer lands after the second user row. + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(71, 'user', 'first')] + : reads === 2 + ? [messageRow(72, 'user', 'second'), messageRow(73, 'assistant', 'first answer, late', { job_id: 'step-1' })] + : [messageRow(74, 'assistant', 'second answer', { job_id: 'step-2' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + await new Promise((r) => setTimeout(r, 50)) + await chat.stop() + await first + await chat.sendMessage('second') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'first', 'row-71'], + ['user', 'second', 'row-72'], + ['assistant', 'first answer, late', 'row-73'], + ['assistant', 'second answer', 'row-74'] + ]) + }) + + test('a failure handler answer is attributed to the turn', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-1') + ? json({ flow_status: { modules: [{ job: 'step-1' }], failure_module: { job: 'handler-1' } } }) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? (reads++, json([messageRow(81, 'user', 'hi'), messageRow(82, 'assistant', 'Sorry: boom', { job_id: 'handler-1', success: false })])) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.success, m.serverId])).toEqual([ + ['user', 'hi', true, 'row-81'], + ['assistant', 'Sorry: boom', false, 'row-82'] + ]) + }) + + test('deleting the current local conversation mid-turn leaves nothing behind', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 300)) + const id = chat.getState().conversationId! + await chat.deleteConversation(id) + await turn + await new Promise((r) => setTimeout(r, 400)) + expect(chat.getState().conversations).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages).toEqual([]) + expect([...storage.data.values()].join('')).not.toContain('hello') + }) + + test('viewing an older local conversation does not reorder history', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: 'ok' }]) : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + await chat.sendMessage('older') + const older = chat.getState().conversationId! + chat.newConversation() + await chat.sendMessage('newer') + const newer = chat.getState().conversationId! + await chat.selectConversation(older) + await new Promise((r) => setTimeout(r, 400)) + const again = createChat(options({ token: 'tok', storage }, fetch)) + expect((await again.loadConversations()).map((c) => c.id)).toEqual([newer, older]) + }) + + test('destroying the chat mid-turn leaves it idle', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + expect(chat.getState().status).toBe('streaming') + chat.destroy() + await turn + expect(chat.getState().status).toBe('idle') + expect(chat.getState().messages.every((m) => !m.pending)).toBe(true) + }) + + test('destroying the chat during a local turn keeps what it showed', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.destroy() + await turn + const again = createChat(options({ token: 'tok', storage }, fetch)) + await again.loadConversations() + expect(again.getState().conversations.map((c) => c.id)).toEqual([id]) + await again.selectConversation(id) + expect(again.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('switching conversations keeps what a local turn showed so far', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.newConversation() + await turn + expect(chat.getState().messages).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('answers from the flow result when server history keeps failing', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('down', 503) : undefined) + ) + const chat = createChat(options({ history: 'server' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + }) + + test('falls back to local history when the credential cannot read conversations', async () => { + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' ? text('forbidden', 403) : undefined + ) + const chat = createChat(options({}, fetch)) + expect(chat.getState().history).toBe('server') + await chat.loadConversations() + expect(chat.getState().history).toBe('local') + + const explicit = createChat(options({ history: 'server' }, fetch)) + await expect(explicit.loadConversations()).rejects.toThrow('403') + expect(explicit.getState().history).toBe('server') + }) +}) diff --git a/chat-sdk/test/config.test.ts b/chat-sdk/test/config.test.ts new file mode 100644 index 0000000000..2289f20c89 --- /dev/null +++ b/chat-sdk/test/config.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { resolveConfig } from '../src/config' + +const g = globalThis as { process?: unknown; ctx?: unknown; location?: unknown } +const originalProcess = g.process + +afterEach(() => { + g.process = originalProcess + delete g.ctx + delete g.location +}) + +describe('resolveConfig', () => { + test('explicit token defaults history to local; a session defaults to server', () => { + const base = { flowPath: 'f/a/b', baseUrl: 'http://wm.test/', workspace: 'ws' } + expect(resolveConfig({ ...base, token: 't' }).history).toBe('local') + expect(resolveConfig(base).history).toBe('server') + expect(resolveConfig({ ...base, token: 't', history: 'server' }).historyExplicit).toBe(true) + }) + + test('reads the sandboxed raw app env the wrapper injects', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: 'sdk-token', history: 'server' }) + }) + + test('keeps the raw app token off another instance', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://other.test', workspace: 'ws' }).token).toBeUndefined() + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://wm.test' }).token).toBe('sdk-token') + }) + + test('reads the unsandboxed raw app context and uses the page origin', () => { + g.ctx = { ctx: { username: 'admin' }, workspace: 'ws' } + g.location = { origin: 'http://wm.test' } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: undefined }) + }) + + test('refuses an opaque origin without an SDK token', () => { + g.ctx = { workspace: 'ws' } + g.location = { origin: 'null' } + expect(() => resolveConfig({ flowPath: 'f/a/b' })).toThrow('frontend SDK scopes') + }) +}) diff --git a/chat-sdk/test/follow.test.ts b/chat-sdk/test/follow.test.ts new file mode 100644 index 0000000000..19af6baf38 --- /dev/null +++ b/chat-sdk/test/follow.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from 'bun:test' +import { WindmillChatApi } from '../src/api' +import { followJob } from '../src/follow' +import { fetchMock, ndjson, sse } from './support' + +test('a retried streaming step reports its offset as lost before the new sub-job is followed', async () => { + let streams = 0 + const { fetch } = fetchMock((c) => { + if (!c.url.pathname.endsWith('/jobs_u/getupdate_sse/job-1')) return undefined + streams++ + if (streams === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'a' }), stream_offset: 3, flow_stream_job_id: 'agent-1' }, + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'b' }), stream_offset: 4, flow_stream_job_id: 'agent-2' } + ]) + } + return sse([{ type: 'update', stream_offset: 1, flow_stream_job_id: 'agent-2', completed: true, only_result: 'ok' }]) + }) + const api = new WindmillChatApi({ baseUrl: 'http://wm.test', workspace: 'ws', token: 'tok', fetch }) + const offsets: (number | undefined)[] = [] + for await (const _ of followJob(api, 'job-1', { onOffset: (o) => offsets.push(o) })) { + // A resumer that stored offset 3 must not reuse it against agent-2. + } + expect(offsets).toEqual([3, undefined, 1]) +}) diff --git a/chat-sdk/test/react.test.tsx b/chat-sdk/test/react.test.tsx new file mode 100644 index 0000000000..75773ad0f2 --- /dev/null +++ b/chat-sdk/test/react.test.tsx @@ -0,0 +1,103 @@ +import { GlobalRegistrator } from '@happy-dom/global-registrator' +// Test files share one process: the DOM globals must not outlive this file. +GlobalRegistrator.register() + +import { afterAll, describe, expect, test } from 'bun:test' +import React, { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { useWindmillChat, type UseWindmillChat } from '../src/react' +import type { ChatOptions } from '../src/types' +import { fetchMock, memoryStorage } from './support' + +afterAll(() => GlobalRegistrator.unregister()) + +const base: ChatOptions = { + flowPath: 'f/chat/agent', + baseUrl: 'http://wm.test', + workspace: 'ws', + fetch: fetchMock().fetch, + storage: memoryStorage() +} + +/** Renders the hook and hands back what it returned, rerendering with new options on demand. */ +function mountHook() { + let latest: UseWindmillChat | undefined + const Probe = (props: ChatOptions) => { + latest = useWindmillChat(props) + return null + } + const root: Root = createRoot(document.createElement('div')) + const render = (props: ChatOptions) => { + act(() => root.render()) + return latest! + } + return { render, unmount: () => act(() => root.unmount()) } +} + +describe('useWindmillChat', () => { + test('a credential change is a new chat; a new closure for the same credential is not', () => { + const { render, unmount } = mountHook() + const a = render({ ...base, token: 'user-a' }).chat + expect(render({ ...base, token: 'user-a' }).chat).toBe(a) + const b = render({ ...base, token: 'user-b' }).chat + expect(b).not.toBe(a) + + const fn1 = render({ ...base, token: () => 'fn-1' }).chat + expect(fn1).not.toBe(b) + expect(render({ ...base, token: () => 'fn-2' }).chat).toBe(fn1) + + const session = render({ ...base }).chat + expect(session).not.toBe(fn1) + const fn3 = render({ ...base, token: () => 'fn-3' }).chat + expect(fn3).not.toBe(session) + expect(render({ ...base, token: () => 'fn-3', storageKey: 'someone-else' }).chat).not.toBe(fn3) + unmount() + }) + + test('the latest inputs go with the next message', async () => { + const { fetch, calls } = fetchMock((c) => (c.method === 'POST' ? new Response('job-1') : undefined)) + const { render, unmount } = mountHook() + render({ ...base, fetch, token: 'tok', inputs: { docId: 'first' } }) + const hook = render({ ...base, fetch, token: 'tok', inputs: { docId: 'second' } }) + // The run's stream never answers here; only the request matters. + void hook.sendMessage('hi', { inputs: { extra: true } }).catch(() => {}) + await new Promise((r) => setTimeout(r, 20)) + expect(calls.find((c) => c.method === 'POST')?.body).toEqual({ docId: 'second', extra: true, user_message: 'hi' }) + hook.chat.destroy() + // A key the latest render no longer passes is gone from the next message. + const cleared = render({ ...base, fetch, token: 'tok', inputs: {} }) + void cleared.sendMessage('again').catch(() => {}) + await new Promise((r) => setTimeout(r, 20)) + expect(calls.filter((c) => c.method === 'POST')[1]?.body).toEqual({ user_message: 'again' }) + unmount() + }) + + test('the latest render’s run callback starts the next turn', async () => { + const { render, unmount } = mountHook() + const started: string[] = [] + const runner = (name: string) => async () => { + started.push(name) + throw new Error('stop here') + } + const first = render({ ...base, token: 'tok', run: runner('first') }) + const second = render({ ...base, token: 'tok', run: runner('second') }) + expect(second.chat).toBe(first.chat) + await act(() => second.sendMessage('hi').catch(() => {})) + expect(started).toEqual(['second']) + // Dropping the runner means the deployed flow again: a different chat. + expect(render({ ...base, token: 'tok' }).chat).not.toBe(first.chat) + unmount() + }) + + test('a token function is read through a ref, so the latest closure serves the next request', async () => { + const { fetch, calls } = fetchMock((c) => (c.url.pathname.includes('/flow_conversations/list') ? new Response('[]') : undefined)) + const { render, unmount } = mountHook() + const first = render({ ...base, fetch, history: 'server', token: () => 'first' }) + await act(() => first.loadConversations()) + const second = render({ ...base, fetch, history: 'server', token: () => 'second' }) + expect(second.chat).toBe(first.chat) + await act(() => second.loadConversations()) + expect(calls.map((c) => c.headers.authorization)).toEqual(['Bearer first', 'Bearer second']) + unmount() + }) +}) diff --git a/chat-sdk/test/stream.test.ts b/chat-sdk/test/stream.test.ts new file mode 100644 index 0000000000..84f2502133 --- /dev/null +++ b/chat-sdk/test/stream.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from 'bun:test' +import { readServerSentEvents } from '../src/api' +import { createStreamEventParser, parseStreamEvents } from '../src/stream' +import { ndjson } from './support' + +describe('parseStreamEvents', () => { + test('keeps agent events and skips other lines', () => { + const events = parseStreamEvents( + ndjson( + { type: 'token_delta', content: 'Hi' }, + { type: 'reasoning_token_delta', content: 'thinking' }, + { type: 'something_else', content: 'x' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true } + ) + 'not json\n' + ) + expect(events.map((e) => e.type)).toEqual(['token_delta', 'reasoning_token_delta', 'tool_result']) + }) +}) + +describe('createStreamEventParser', () => { + test('holds an incomplete line until the rest arrives', () => { + const parser = createStreamEventParser() + const line = JSON.stringify({ type: 'token_delta', content: 'Hello' }) + expect(parser.push(line.slice(0, 10))).toEqual([]) + expect(parser.push(line.slice(10) + '\n' + '{"type":"token_delta",')).toEqual([ + { type: 'token_delta', content: 'Hello' } + ]) + expect(parser.push('"content":"!"}')).toEqual([]) + expect(parser.flush()).toEqual([{ type: 'token_delta', content: '!' }]) + }) +}) + +describe('readServerSentEvents', () => { + test('splits frames that straddle chunks and normalizes CRLF', async () => { + const chunks = ['data: {"a":1}\r\n\r\ndata: {"b"', ':2}\n\ndata: first\ndata: second\n\n', 'data: {"c":3}'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['{"a":1}', '{"b":2}', 'first\nsecond', '{"c":3}']) + }) + + test('keeps a CRLF split across chunks from ending the event', async () => { + const chunks = ['data: first\r', '\ndata: second\r\n\r\ndata: last\r'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['first\nsecond', 'last']) + }) +}) diff --git a/chat-sdk/test/support.ts b/chat-sdk/test/support.ts new file mode 100644 index 0000000000..c794e50f3d --- /dev/null +++ b/chat-sdk/test/support.ts @@ -0,0 +1,101 @@ +import type { FetchLike, StorageLike } from '../src/types' + +export interface RecordedCall { + method: string + url: URL + headers: Record + body: unknown +} + +export type Route = (call: RecordedCall) => Response | Promise | undefined + +/** A fetch whose responses come from the first route that answers; every call is recorded. */ +export function fetchMock(...routes: Route[]): { fetch: FetchLike; calls: RecordedCall[] } { + const calls: RecordedCall[] = [] + const fetch: FetchLike = async (input, init) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url) + const call: RecordedCall = { + method: init?.method ?? 'GET', + url, + headers: Object.fromEntries( + Object.entries((init?.headers as Record) ?? {}).map(([k, v]) => [k.toLowerCase(), v]) + ), + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined + } + calls.push(call) + for (const route of routes) { + const res = await route(call) + if (res) return res + } + return new Response(`no route for ${call.method} ${url.pathname}`, { status: 404 }) + } + return { fetch, calls } +} + +export function json(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +export function text(value: string, status = 200): Response { + return new Response(value, { status }) +} + +/** A `text/event-stream` body carrying one `data:` frame per event. */ +export function sse(events: object[]): Response { + return new Response(events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(''), { + status: 200, + headers: { 'content-type': 'text/event-stream' } + }) +} + +/** Like `sse`, but a number in the list pauses that many milliseconds before the next frame. */ +export function sseTimed(events: (object | number)[]): Response { + const encoder = new TextEncoder() + const body = new ReadableStream({ + async start(controller) { + for (const e of events) { + if (typeof e === 'number') await new Promise((r) => setTimeout(r, e)) + else controller.enqueue(encoder.encode(`data: ${JSON.stringify(e)}\n\n`)) + } + controller.close() + } + }) + return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }) +} + +export function ndjson(...events: object[]): string { + return events.map((e) => JSON.stringify(e)).join('\n') + '\n' +} + +export function memoryStorage(): StorageLike & { data: Map } { + const data = new Map() + return { + data, + getItem: (k) => data.get(k) ?? null, + setItem: (k, v) => void data.set(k, v), + removeItem: (k) => void data.delete(k) + } +} + +export function messageRow( + seq: number, + type: 'user' | 'assistant' | 'tool', + content: string, + extra: Record = {} +) { + return { + id: `row-${seq}`, + conversation_id: 'conv', + message_type: type, + content, + job_id: null, + created_at: '2026-01-01T00:00:00Z', + created_seq: seq, + step_name: null, + success: true, + ...extra + } +} diff --git a/chat-sdk/tsconfig.build.json b/chat-sdk/tsconfig.build.json new file mode 100644 index 0000000000..b8fc4fa324 --- /dev/null +++ b/chat-sdk/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*"], + "compilerOptions": { + "types": [], + "noEmit": false, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "rootDir": "src" + } +} diff --git a/chat-sdk/tsconfig.json b/chat-sdk/tsconfig.json new file mode 100644 index 0000000000..fbfee1f0d5 --- /dev/null +++ b/chat-sdk/tsconfig.json @@ -0,0 +1,15 @@ +{ + "include": ["src/**/*", "test/**/*"], + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "types": ["bun"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} diff --git a/chat-sdk/tsdown.config.ts b/chat-sdk/tsdown.config.ts new file mode 100644 index 0000000000..98b23c536e --- /dev/null +++ b/chat-sdk/tsdown.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/index.ts', 'src/react.ts', 'src/ai-sdk.ts', 'src/assistant-ui.ts'], + format: ['esm', 'cjs'], + dts: false, + external: ['react', 'ai', '@assistant-ui/react'], + target: 'es2020', +}) diff --git a/cli/TESTING.md b/cli/TESTING.md index 235a95c4b1..df0311677d 100644 --- a/cli/TESTING.md +++ b/cli/TESTING.md @@ -22,6 +22,12 @@ Pure local tests — no backend, no database. Uses `bunfig.unit.toml` (no preloa Examples: `git_unit`, `lint_command_unit`, `tar_creation_unit`, `workspace_conflicts_unit` +`mock.module()` mocks the module for the whole `bun test` process, not for the file +that installs it, and `mock.restore()` does not undo it. A file that mocks a module +must hand back its real exports in `afterAll` (see +`schedule_push_permissioned_as_unit`), or it silently rewires whichever file runs +next — and the run order is the directory's, so it differs between Linux and Windows. + ### Integration tests Require a running backend and PostgreSQL. The `setup.ts` preload builds the backend @@ -29,6 +35,28 @@ binary and starts a shared backend instance. Examples: `sync_pull_push`, `dev_server`, `standalone_commands` +## Module mocks + +`mock.module` replaces a module for the **whole process**, and it does reach modules that +were already imported — a stub one file installs lands on a consumer an earlier file +loaded. + +Handing the module back in `afterAll` is not a reliable undo. Files do run one at a time +(a root-level `afterAll` completes before the next file's body evaluates), so it looks +like it should be — but stubbing `bundle.ts` and restoring it that way still left +`raw_app_svelte_plugin_unit.test.ts` asserting against an empty bundle, green on Linux +and red on Windows, where the `readdir` file order differs. Treat a stub as permanent for +the run. + +So the rule is about what you stub, not how you clean up: **stub only a module no other +in-process suite imports.** Check with `grep -rl "" test/` before reaching +for one. A suite that drives the CLI through a spawned process is out of reach of a +module mock and doesn't count. + +`raw_app_push_policy_unit.test.ts` is the worked example: it stubs `gen/services.gen.ts`, +which passes the rule because nothing else in `test/` imports the three API functions it +replaces, and deliberately does not stub `bundle.ts`, which failed it. + ## AI Benchmark Caveats The repo-level benchmark CLI lives under `ai_evals/`, but it currently depends on diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0f46ec76be..0d4ad634c8 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + getWmillYamlPath, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; import lintCommand from "./lint.ts"; @@ -21,6 +25,7 @@ import newCommand from "./new.ts"; import generateAgentsCommand from "./generate_agents.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; export interface AppFile { @@ -114,9 +119,10 @@ export function isExecutionModeAnonymous(app: any) { export function isExecutionModeGuest(app: any) { return app?.["policy"]?.["execution_mode"] == "guest"; } -export type AppExecutionMode = "anonymous" | "guest" | "publisher"; +export type AppExecutionMode = "anonymous" | "guest" | "publisher" | "viewer"; /** The access mode is the one policy field a tracked app keeps, as `public` (anonymous) - * or `guests` (guest); the rest of the policy is regenerated on push. */ + * or `guests` (guest); the rest of the policy is preserved from the deployed app on + * push (see `generatingPolicy`). */ export function markAccessFromPolicy(app: any) { if (isExecutionModeAnonymous(app)) { app.public = true; @@ -124,14 +130,35 @@ export function markAccessFromPolicy(app: any) { app.guests = true; } } -export function executionModeFromAppFile(app: any): AppExecutionMode { +/** The mode the tracked file states, or `undefined` when it states none — the + * normal case, since a pull writes only the two open-access markers. `viewer` + * and `publisher` have no marker of their own, so a file can only name them + * through a policy block it was hand-written with. */ +function statedExecutionMode(app: any): AppExecutionMode | undefined { if (app?.["public"] ?? isExecutionModeAnonymous(app)) { return "anonymous"; } if (app?.["guests"] ?? isExecutionModeGuest(app)) { return "guest"; } - return "publisher"; + const mode = app?.["policy"]?.["execution_mode"]; + return mode === "viewer" || mode === "publisher" ? mode : undefined; +} + +/** The mode this push deploys under. A file that states one is authoritative, in + * both directions. Otherwise the two open-access markers are all it says, so + * their absence closes a deployed `anonymous`/`guest` app back down to + * `publisher` — while a deployed `viewer` is not a grant those markers revoke, + * so it carries over rather than widening to `publisher`. */ +export function executionModeForPush( + localApp: any, + deployedPolicy: Policy | undefined, +): AppExecutionMode { + const stated = statedExecutionMode(localApp); + if (stated) { + return stated; + } + return deployedPolicy?.execution_mode === "viewer" ? "viewer" : "publisher"; } export async function pushApp( workspace: string, @@ -156,12 +183,9 @@ export async function pushApp( //ignore } - let remoteOnBehalfOf: string | undefined; - let remoteOnBehalfOfEmail: string | undefined; - if (app?.policy) { - remoteOnBehalfOf = app.policy.on_behalf_of; - remoteOnBehalfOfEmail = app.policy.on_behalf_of_email; - } + // `app.policy` is cleared a few lines down, so capture it first: it is the + // base the regenerated policy is built on. + const deployedPolicy: Policy | undefined = app?.policy; markAccessFromPolicy(app); // console.log(app); @@ -176,20 +200,18 @@ export async function pushApp( const localApp = (await yamlParseFile(path)) as AppFile; replaceInlineScripts(localApp.value, localPath, true); - await generatingPolicy(localApp, remotePath, executionModeFromAppFile(localApp)); - - const preserveFields: { preserve_on_behalf_of?: boolean } = {}; - if (permissionedAsContext?.userIsAdminOrDeployer) { - if (app) { - if (localApp.policy && remoteOnBehalfOf) { - (localApp.policy as any).on_behalf_of = remoteOnBehalfOf; - (localApp.policy as any).on_behalf_of_email = remoteOnBehalfOfEmail; - preserveFields.preserve_on_behalf_of = true; - log.info(`Preserving ${remoteOnBehalfOfEmail ?? remoteOnBehalfOf} as permissioned_as for app ${remotePath}`); - } - } - // On create: backend applies folder defaults - } + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext + ); + await generatingPolicy( + localApp, + remotePath, + executionModeForPush(localApp, deployedPolicy), + basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of) + ); // extra_perms goes through /acls/* — strip from the body so a perms-only // edit never bumps the app version (see applyExtraPermsDiff for details). @@ -246,18 +268,76 @@ export async function pushApp( export async function generatingPolicy( app: any, path: string, - executionMode: AppExecutionMode + executionMode: AppExecutionMode, + base: Policy | undefined ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updatePolicy(app.value, undefined); - app.policy.execution_mode = executionMode; + app.policy = await windmillUtils.updatePolicy(app.value, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; } } +/** What the regenerated policy starts from: the deployed one, so a push keeps + * settings the tracked file doesn't record; on a first push, whatever the file + * states. The run identity rides along only when `claimsOnBehalfOf` — never + * from the file, never from a pusher who may not preserve one, since `wmill` + * is regularly pointed at servers older than the rewrite that would fix it. */ +export function basePolicy( + localApp: any, + deployedPolicy: Policy | undefined, + claimsOnBehalfOf: boolean +): Policy | undefined { + const stated = deployedPolicy ?? (localApp?.policy as Policy | undefined); + if (!stated || claimsOnBehalfOf) { + return stated; + } + const base: Policy = { ...stated }; + delete base.on_behalf_of; + delete base.on_behalf_of_email; + return base; +} + +/** Claim the run-as identity the regenerated policy carries over from the + * deployed app. Only a deployed identity may be claimed, never one the tracked + * file states — a repo doesn't get to pick who an app runs as. Without the flag + * the backend rewrites `on_behalf_of` to whoever is pushing, and it only honors + * the flag for an admin or a `wm_deployers` member, so a caller who is neither + * doesn't get to claim it here either. */ +export function preserveOnBehalfOfFields( + remotePath: string, + deployedPolicy: Policy | undefined, + permissionedAsContext: PermissionedAsContext | undefined +): { preserve_on_behalf_of?: boolean } { + const onBehalfOf = deployedPolicy?.on_behalf_of; + if (!permissionedAsContext?.userIsAdminOrDeployer || !onBehalfOf) { + return {}; + } + log.info( + `Preserving ${deployedPolicy?.on_behalf_of_email ?? onBehalfOf} as permissioned_as for app ${remotePath}` + ); + return { preserve_on_behalf_of: true }; +} + +/** The policy is written wholesale by the deploy, so the fields it does not + * derive from the tracked sources have to survive the trip. The policy builder + * has already recomputed what it can — the triggerables on both paths, plus the + * S3 rules on the low-code one, which `updateRawAppPolicy` has no equivalent of + * and so carries over. This sets the two left: the access mode, and the legacy + * `triggerables`, which still grant execution (the backend folds them into + * `triggerables_v2` at run time) and so are dropped rather than carried, or a + * deployed app would keep being able to run runnables this push removed. */ +export function finalizeDerivedPolicy( + policy: Policy, + executionMode: AppExecutionMode +) { + policy.triggerables = undefined; + policy.execution_mode = executionMode; +} + async function list(opts: GlobalOptions & { includeDraftOnly?: boolean; json?: boolean }) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -426,10 +506,23 @@ async function push( absoluteFilePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } else { - await pushApp(workspace.workspaceId, remotePath, absoluteFilePath); + await pushApp( + workspace.workspaceId, + remotePath, + absoluteFilePath, + undefined, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), + ); log.info(colors.bold.underline.green("App pushed")); } } diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 901a47593f..8d82edb9df 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -1,6 +1,9 @@ import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; @@ -17,13 +20,22 @@ import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { type AppExecutionMode, - executionModeFromAppFile, + basePolicy, + executionModeForPush, + finalizeDerivedPolicy, markAccessFromPolicy, + preserveOnBehalfOfFields, replaceInlineScripts, repopulateFields, } from "./app.ts"; +import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER, RECORDINGS_FOLDER } from "./app_metadata.ts"; +import { + NEVER_DEPLOYED_DIRS, + NEVER_DEPLOYED_FILES, +} from "../../utils/app_files.ts"; import { writeIfChanged } from "../../utils/utils.ts"; import { yamlOptions } from "../sync/sync.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; @@ -316,13 +328,11 @@ async function collectAppFiles( const relativePath = basePath + entry.name; if (entry.isDirectory()) { - // Skip the runnables, node_modules, and sql_to_apply subfolders + // The backend folder deploys as `value.runnables`, not as a bundled + // file; the rest reach the server through no channel at all. if ( entry.name === APP_BACKEND_FOLDER || - entry.name === "node_modules" || - entry.name === "dist" || - entry.name === ".claude" || - entry.name === "sql_to_apply" + NEVER_DEPLOYED_DIRS.has(entry.name) ) { continue; } @@ -334,13 +344,11 @@ async function collectAppFiles( } await readDirRecursive(fullPath + SEP, relativePath + "/"); } else if (entry.isFile()) { - // Skip generated/metadata files that shouldn't be part of the app + // `raw_app.yaml` deploys as the request's metadata rather than as a + // bundled file; the rest reach the server through no channel at all. if ( entry.name === "raw_app.yaml" || - entry.name === "package-lock.json" || - entry.name === "DATATABLES.md" || - entry.name === "AGENTS.md" || - entry.name === "wmill.d.ts" + NEVER_DEPLOYED_FILES.has(entry.name) ) { continue; } @@ -360,6 +368,7 @@ export async function pushRawApp( localPath: string, message?: string, defaultTs: "bun" | "deno" = "bun", + permissionedAsContext?: PermissionedAsContext, ): Promise { if (alreadySynced.includes(localPath)) { return; @@ -376,6 +385,10 @@ export async function pushRawApp( } catch { //ignore } + // `app.policy` is cleared a few lines down, so capture it first. `raw_app.yaml` + // records none of the policy, so anything the deploy drawer set is only here. + const deployedPolicy: Policy | undefined = app?.policy; + markAccessFromPolicy(app); // console.log(app); if (app) { @@ -424,10 +437,21 @@ export async function pushRawApp( // Create a temporary app object for policy generation const appForPolicy = { ...localApp, runnables }; + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext, + ); await generatingPolicy( appForPolicy, remotePath, - executionModeFromAppFile(localApp), + executionModeForPush(localApp, deployedPolicy), + basePolicy( + localApp, + deployedPolicy, + !!preserveFields.preserve_on_behalf_of, + ), ); const files = await collectAppFiles(localPath); @@ -482,6 +506,7 @@ export async function pushRawApp( path: remotePath, summary: localApp.summary, policy: appForPolicy.policy, + ...preserveFields, deployment_message: message, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, @@ -532,14 +557,12 @@ export async function generatingPolicy( app: any, path: string, executionMode: AppExecutionMode, + base: Policy | undefined, ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updateRawAppPolicy( - app.runnables, - app.policy, - ); - app.policy.execution_mode = executionMode; + app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; @@ -564,6 +587,10 @@ async function pushRawAppCommand( filePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7a383fcfeb..863690ce7d 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -4,7 +4,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; -import { dirname, sep as SEP } from "node:path"; +import { dirname, sep as SEP, resolve as pathResolve } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; @@ -21,11 +21,16 @@ import { } from "../../core/context.ts"; import { resolve, track_job, pollForJobResult } from "../script/script.ts"; import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; -import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + SyncOptions, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { collectPathScriptPaths, replaceInlineScripts, @@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo if (!validatePath(remotePath)) { return; } + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); - await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); + await pushFlow( + workspace.workspaceId, + remotePath, + filePath, + opts.message, + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) + ); log.info(colors.bold.underline.green("Flow pushed")); } diff --git a/cli/src/commands/hub/hub.ts b/cli/src/commands/hub/hub.ts index f752147811..1a9c89680e 100644 --- a/cli/src/commands/hub/hub.ts +++ b/cli/src/commands/hub/hub.ts @@ -20,6 +20,9 @@ interface HubResourceType { // Absent from hubs predating the column, so a missing value is "ordinary type", // not "unset it". format_extension?: string | null; + // Null where nobody named the type, and absent from hubs predating the field, which + // leaves a stored name alone rather than clearing it. + display_name?: string | null; } export async function pull(opts: GlobalOptions) { @@ -120,7 +123,9 @@ export async function pull(opts: GlobalOptions) { deepEqual(y.schema, x.schema) && y.description === x.description && (y.is_fileset ?? false) === (x.is_fileset ?? false) && - (y.format_extension ?? null) === (x.format_extension ?? null) + (y.format_extension ?? null) === (x.format_extension ?? null) && + (x.display_name === undefined || + (y.display_name ?? null) === x.display_name) ) ) { log.info("skipping " + x.name + " (same as current)"); diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts index f6a35429ef..d031c16b3f 100644 --- a/cli/src/commands/lint/lint.ts +++ b/cli/src/commands/lint/lint.ts @@ -31,10 +31,18 @@ import { import { isFlowInlineScriptPath, isAppInlineScriptPath, - isRawAppPath, + isFolderResourcePathAnyFormat, getFolderSuffix, + getScriptBasePathFromModulePath, } from "../../utils/resource_folders.ts"; -import { exts } from "../script/script.ts"; +import { isFilesetResource } from "../../utils/utils.ts"; +import { + exts, + findContentFile, + hasScriptExt, + isModuleEntryMetadata, + UnresolvableScriptContentFileError, +} from "../script/script.ts"; interface LintOptions extends GlobalOptions { json?: boolean; @@ -67,6 +75,9 @@ export interface LintReport { const YAML_FILE_REGEX = /\.ya?ml$/i; const NATIVE_TRIGGER_REGEX = /\.[^.]+_native_trigger\.ya?ml$/i; +// The metadata suffixes `findContentFile` resolves a flat script from. `.yml` is +// deliberately absent, since the push does not accept it there either. +const FLAT_SCRIPT_METADATA_REGEX = /\.script\.(yaml|json|lock)$/; function normalizePath(p: string): string { return p.replaceAll(SEP, "/"); @@ -643,6 +654,83 @@ export async function checkMissingLocks( return issues; } +/** + * Whether a path is a script's own metadata, as opposed to metadata the push + * deploys through some parent: a folder resource's inline scripts, a fileset's + * children (arbitrarily named, so one may be spelled exactly like a script's + * metadata) and the files of a module or dbt bundle all belong to that parent. + * + * Takes the path as the SYNC ROOT spells it, like the push. Relative to the + * lint target the enclosing folder is gone whenever the target IS that folder; + * absolute, the classifiers match their suffixes ANYWHERE in the string, so a + * checkout under `acme.app` reads as one app and nothing is ever reported. + */ +function isStandaloneScriptMetadata(rootedPath: string): boolean { + // Both suffix formats, because the dotted/non-dotted setting is read from the + // invocation directory and an explicit lint target may not share it. + if ( + isFolderResourcePathAnyFormat(rootedPath) || + isFilesetResource(rootedPath) + ) { + return false; + } + // A module folder keeps its metadata inside itself (`__mod/script.yaml`), + // which is standalone even though every other path under `__mod/` is not. + if (isModuleEntryMetadata(rootedPath)) return true; + if (getScriptBasePathFromModulePath(rootedPath) !== undefined) return false; + return FLAT_SCRIPT_METADATA_REGEX.test(rootedPath); +} + +/** + * `findContentFile` quotes the paths it was given back in its errors, so the + * lint target's own prefix comes off them again. Anchored at a path start: a + * plain substring replace of `f/` also eats the one inside `conf/`, mangling + * the very filename the message is telling the reader to delete. + */ +function relativizeMessage(message: string, prefix: string): string { + if (!prefix) return message; + const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return message.replaceAll(new RegExp(`(^|[\\s(])${escaped}/`, "g"), "$1"); +} + +/** + * Script metadata files that cannot be paired with exactly one content file: + * the push refuses those, and no metadata format makes them deployable, so the + * inactive twin of a format switch (`foo.script.json` in a yaml repo) is dead + * weight worth reporting even though the push skips it rather than refusing it. + * + * Resolved through `findContentFile` rather than by probing `exts` directly, so + * lint and push agree on what counts as paired: a dbt project's descriptor is + * optional and its absence is not an orphan, while two content files beside one + * metadata file is just as undeployable as none. It classifies what it is given + * and looks under `syncRoot`, so where the command was invoked from is not part + * of the answer. + */ +async function checkOrphanScriptMetadata( + syncRoot: string, + prefix: string, + metadataPaths: string[], +): Promise { + const issues: FileIssue[] = []; + for (const metadataPath of metadataPaths) { + const rootedPath = prefix ? `${prefix}/${metadataPath}` : metadataPath; + try { + await findContentFile(rootedPath, syncRoot); + } catch (e) { + if (!(e instanceof UnresolvableScriptContentFileError)) { + log.debug(`Failed to resolve content file for ${rootedPath}: ${e}`); + continue; + } + issues.push({ + path: metadataPath, + target: "script", + errors: [relativizeMessage(e.message, prefix)], + }); + } + } + return issues; +} + export async function runLint( opts: LintOptions, directory?: string, @@ -674,8 +762,16 @@ export async function runLint( const root = await FSFSElement(targetDirectory, [], false); const validator = new WindmillYamlValidator(); + // Walked paths are relative to the lint target; this puts them back the way + // the sync root spells them, which is what the two below are written against. + const syncRoot = await findSyncRoot(targetDirectory); + const metadataPrefix = normalizePath( + path.relative(syncRoot, targetDirectory), + ); + const warnings: LintWarning[] = []; const issues: FileIssue[] = []; + const scriptMetadataPaths: string[] = []; let scannedFiles = 0; let validatedFiles = 0; let validFiles = 0; @@ -689,6 +785,17 @@ export async function runLint( const normalizedPath = normalizePath(entry.path); scannedFiles += 1; + + // Collected before the YAML filter below: `.script.lock` and `.script.json` + // are metadata too, and both fail the push when nothing pairs with them. + if ( + isStandaloneScriptMetadata( + metadataPrefix ? `${metadataPrefix}/${normalizedPath}` : normalizedPath, + ) + ) { + scriptMetadataPaths.push(normalizedPath); + } + if (!YAML_FILE_REGEX.test(normalizedPath)) { continue; } @@ -727,6 +834,16 @@ export async function runLint( } } + // Unconditional: unlike a missing lock, metadata with no content file fails + // every push, so there is no mode in which it is acceptable. + issues.push( + ...(await checkOrphanScriptMetadata( + syncRoot, + metadataPrefix, + scriptMetadataPaths, + )), + ); + // Check for missing locks if --locks-required is set if (opts.locksRequired) { const lockIssues = await checkMissingLocks(opts, explicitTargetDirectory); @@ -820,6 +937,15 @@ async function lint(opts: LintOptions & { watch?: boolean }, directory?: string) } } +/** + * Whether a changed file can change what a lint run reports: metadata in any of + * its formats, and the content files whose presence is what keeps that metadata + * from being an orphan. + */ +function affectsLint(filename: string): boolean { + return /\.(ya?ml|json|lock)$/i.test(filename) || hasScriptExt(filename); +} + async function lintWatch(opts: LintOptions, directory?: string) { const { watch } = await import("node:fs"); const targetDir = directory ? path.resolve(process.cwd(), directory) : process.cwd(); @@ -842,7 +968,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { let debounce: ReturnType | null = null; watch(targetDir, { recursive: true }, (_event, filename) => { - if (!filename || !filename.toString().endsWith(".yaml") && !filename.toString().endsWith(".yml")) return; + if (!filename || !affectsLint(filename.toString())) return; if (debounce) clearTimeout(debounce); debounce = setTimeout(runAndReport, 300); }); @@ -853,7 +979,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { const command = new Command() .description( - "Validate Windmill flow, schedule, and trigger YAML files in a directory", + "Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file", ) .arguments("[directory:string]") .option("--json", "Output results in JSON format") diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index 5f9e713f42..f6173a28bb 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -413,7 +413,13 @@ export function parseMuteAnnotations(content: string): { // Comment prefix for `volume:` annotations. Deliberately NOT `commentPrefix` // above (which returns `--` for SQL): volume annotations are only recognized for // the languages the backend/frontend recognize them for — mirrors -// `asset_inference.rs:comment_prefix` and `infer.ts:getCommentPrefix` (SQL → none). +// `asset_inference.rs:comment_prefix` and `infer.ts:getCommentPrefix` (SQL → none), +// minus `php`. Those two recognize PHP, but a PHP script cannot reach this map: it +// has no wasm asset parser, so `fallbackParse` handles it and that scan breaks on +// the mandatory ` { const project = basePath + "__dbt/dbt_project.yml"; - return (await stat(project).then(() => true).catch(() => false)) + const onDisk = baseDir ? path.join(baseDir, project) : project; + return (await stat(onDisk).then(() => true).catch(() => false)) ? project : undefined; } @@ -1137,7 +1141,14 @@ async function readScriptContent(filePath: string): Promise { } } -export async function findContentFile(filePath: string) { +/** + * The script file `filePath`'s metadata belongs to. `baseDir`, when given, is + * where the disk lookups happen, leaving `filePath` classified as written: the + * layout helpers below match their suffixes ANYWHERE in a path, so a caller + * that prefixed a checkout named `repo__mod` would have it read as the module. + */ +export async function findContentFile(filePath: string, baseDir?: string) { + const onDisk = (p: string) => (baseDir ? path.join(baseDir, p) : p); // Folder layout: __mod/script.yaml -> __mod/script.ts const isModuleFolderMeta = isModuleEntryMetadata(filePath); const toCandidate = (ext: string) => @@ -1161,7 +1172,7 @@ export async function findContentFile(filePath: string) { const validCandidates = ( await Promise.all( candidates.map((x) => { - return stat(x) + return stat(onDisk(x)) .catch(() => undefined) .then((x) => x?.isFile()) .then((e) => { @@ -1181,6 +1192,7 @@ export async function findContentFile(filePath: string) { const dbtCandidate = toCandidate("__dbt/" + DBT_DESCRIPTOR_NAME); const dbtProject = await collidingDbtProject( dbtCandidate.slice(0, -("__dbt/" + DBT_DESCRIPTOR_NAME).length), + baseDir, ); const nonDbtCandidates = validCandidates.filter((c) => c !== dbtCandidate); if (dbtProject && nonDbtCandidates.length > 0) { diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2702e13714..d75dc81912 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -104,7 +104,7 @@ export async function downloadZip( // from v1 the on-behalf-of address is stripped below, so the tarball sends the // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs - // on flow / script / app rows. Default-off on the server protects cross- + // on script / flow / app / variable rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities // missing in the target workspace; the CLI sync flow explicitly wants them. const baseParams = `&plain_secret=${plainSecrets ?? false @@ -150,7 +150,18 @@ export async function downloadZip( } if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + log.info( + colors.red( + `Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` + + (workspace.name !== workspace.workspaceId + ? ` (resolved from profile '${workspace.name}')` + : "") + + `.\n` + + `Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` + + ` - check 'wmill workspace list' (the 'workspace id' column)\n` + + ` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)` + ) + ); } else { log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); if (body) log.info(colors.red(body)); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index be5d9e5639..138a346594 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -76,7 +76,8 @@ import { } from "../../utils/utils.ts"; import { getEffectiveSettings, - getWorkspaceNames, + inferWsNameFromProfile, + resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, SyncOptions, @@ -85,7 +86,10 @@ import { WorkspaceEntryConfig, } from "../../core/conf.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { preCheckPermissionedAs } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + preCheckPermissionedAs, +} from "../../core/permissioned_as.ts"; import { fromWorkspaceSpecificPath, toWorkspaceSpecificPath, @@ -124,6 +128,8 @@ import { } from "../../utils/metadata.ts"; import { DoubleLinkedDependencyTree, + LocalScripts, + resolvePlaceholdersFromLocal, uploadScripts, } from "../../utils/dependency_tree.ts"; import { @@ -148,6 +154,7 @@ import { generateAppLocksInternal, RECORDINGS_FOLDER, } from "../app/app_metadata.ts"; +import { deploysWithRawApp } from "../../utils/app_files.ts"; import { isFlowPath, isAppPath, @@ -171,6 +178,7 @@ import { isDbtModulePath, isDbtGeneratedPath, isModuleEntryPoint, + scriptPathToRemotePath, getScriptBasePathFromModulePath, hasWrongFormatSuffix, DBT_DESCRIPTOR_NAME, @@ -429,37 +437,6 @@ export function computeWsSpecificFlagOnlyPushes( return out; } -// Resolve workspace name from a --branch override (git branch → workspace name). -// Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch( - opts: SyncOptions, - branchName: string, -): string { - const match = findWorkspaceByGitBranch(opts.workspaces, branchName); - return match ? match[0] : branchName; -} - -// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, -// then --workspace → matching config key (incl. when --base-url is set). Returns -// undefined when no flag-based resolution applies; callers then fall back to -// inferWsNameFromProfile on the resolved workspace profile. -export function resolveWsNameForConfigFromFlags( - opts: SyncOptions & { branch?: string; workspace?: string }, -): string | undefined { - if (opts.branch) { - return resolveWsNameFromBranch(opts, opts.branch); - } - if (opts.workspace) { - // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, - // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. - const validKeys = getWorkspaceNames(opts.workspaces); - if (validKeys.includes(opts.workspace)) { - return opts.workspace; - } - } - return undefined; -} - // Warn if --workspace overrides auto-detected branch or if workspace not in config. function warnWorkspaceOverride( opts: SyncOptions, @@ -507,33 +484,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { return wsName; } -// After resolveWorkspace, infer the workspace config name from the resolved profile -// by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile( - opts: SyncOptions, - profile: { remote: string; workspaceId: string }, -): string | undefined { - if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter( - (k) => k !== "commonSpecificItems", - ); - for (const name of wsNames) { - const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; - if (!entry?.baseUrl) continue; - try { - const entryUrl = new URL(entry.baseUrl).toString(); - const profileUrl = new URL(profile.remote).toString(); - const entryWsId = entry.workspaceId ?? name; - if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { - return name; - } - } catch { - continue; - } - } - return undefined; -} - // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< T extends GlobalOptions & SyncOptions & { repository?: string }, @@ -2072,20 +2022,18 @@ export async function elementsToMap( } if (isRawAppFile(path)) { - // FSFSElement builds paths with the platform separator, while the checks - // below are written with "/": without normalizing, none of them match on - // Windows and the push collector's own exclusions become perpetual diffs. + // FSFSElement builds paths with the platform separator, while + // `deploysWithRawApp` is written with "/": without normalizing it matches + // nothing on Windows and the push collector's own exclusions become + // perpetual diffs. const suffix = path .split(getFolderSuffix("raw_app") + SEP) .pop() ?.replaceAll(SEP, "/"); - if ( - suffix?.startsWith("dist/") || - suffix?.startsWith(RECORDINGS_FOLDER + "/") || - suffix == "wmill.d.ts" || - suffix == "package-lock.json" || - suffix == "DATATABLES.md" - ) { + // A file no push sends is not a change to track. Listing it leaves it + // pending forever — nothing ever uploads it — and pushing it redeploys + // the whole app, reassigning its run-as user, to ship nothing. + if (suffix && !deploysWithRawApp(suffix)) { continue; } } @@ -2597,6 +2545,21 @@ export function preservePendingScriptLocks( } } +// `sync push` never applies the workspace's display name from settings.yaml and +// applies its color only when the local file carries one (see +// pushWorkspaceSettings), so on a push the fields it would not apply must +// compare equal, or the row is listed on every run. +const isWorkspaceSettingsFile = (p: string) => + /^settings(\.[^./\\]+)?\.(yaml|json)$/.test(p); +function stripUnappliedSettingsFields(local: any, remote: any) { + delete local?.name; + delete remote?.name; + if (local?.color == null) { + delete local?.color; + delete remote?.color; + } +} + export async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, @@ -2812,6 +2775,9 @@ export async function compareDynFSElement( delete parsedV?.enabled; delete parsedM2?.enabled; } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(parsedV, parsedM2); + } if (deepEqual(parsedV, parsedM2)) { continue; } @@ -2826,6 +2792,9 @@ export async function compareDynFSElement( delete before?.enabled; delete after?.enabled; } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(after, before); + } if (deepEqual(before, after)) { continue; } @@ -3209,6 +3178,57 @@ export function untrackedDatatableMigrationDeletions< ); } +/** + * The kind of secret-bearing object whose deletion this path is, if it is one. + * + * Classified with the push's own `getTypeStrFromPath`, so this agrees with the switch + * that does the deleting. A fileset child is excluded ahead of it: it can be any file, + * `inner.resource.yaml` included, and deleting one re-pushes the parent resource + * rather than deleting anything. + */ +export function secretBearingObjectKind( + p: string, +): "variable" | "resource" | undefined { + if (isFilesetResource(p)) return undefined; + // The apply loop `continue`s past a `.lock` deletion before reaching the switch, + // so counting one would announce a deletion the push never performs. A raw-app or + // dbt `.lock`, the two that loop does not skip, classifies as its bundle's own kind + // long before the file-resource check, so a plain suffix test is enough here. + if (p.endsWith(".lock")) return undefined; + let typ: string; + try { + typ = getTypeStrFromPath(p); + } catch { + // Not a path the push classifies, so not one it deletes. + return undefined; + } + return typ === "variable" || typ === "resource" ? typ : undefined; +} + +/** The server-side object a secret-bearing file belongs to, so a file resource's two + * files are counted (and reported) as the one resource they delete. */ +function secretBearingObjectPath(p: string): string { + const normalized = p.replaceAll(SEP, "/"); + return secretBearingObjectKind(p) === "resource" + ? removeResourceSuffix(normalized) + : normalized.replace(/\.variable\.(yaml|json)$/, ""); +} + +/** e.g. "2 variables and 1 resource", counted by object rather than by file. */ +export function describeSecretBearingChanges( + changes: { path: string }[], +): string { + const objects = { variable: new Set(), resource: new Set() }; + for (const c of changes) { + const kind = secretBearingObjectKind(c.path); + if (kind) objects[kind].add(secretBearingObjectPath(c.path)); + } + return (["variable", "resource"] as const) + .filter((k) => objects[k].size > 0) + .map((k) => `${objects[k].size} ${k}${objects[k].size > 1 ? "s" : ""}`) + .join(" and "); +} + /** * Whether a pull change removes a local dbt descriptor. A dbt project's * descriptor is optional and the remote spells "this project names none" as @@ -3368,6 +3388,37 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { } } +/** + * Index the checkout's standalone scripts by the remote path a relative import + * resolves to, reusing the content the local/remote diff already read. + * + * Same classification as `addToChangedIfNotExists`: a flow or app inline script + * is not addressable as an import target, and a module bundle is addressed by + * its entry point. + */ +function localScriptsByRemotePath( + localMap: Record, +): LocalScripts { + const byRemotePath: LocalScripts = new Map(); + for (const [p, content] of Object.entries(localMap)) { + if (isScriptModulePath(p)) { + if (!isModuleEntryPoint(p)) continue; + } else if ( + !hasScriptExt(p) || + isDatatableMigrationPath(p) || + isFileResource(p) || + isFilesetResource(p) || + isFlowPath(p) || + isAppPath(p) || + isRawAppPath(p) + ) { + continue; + } + byRemotePath.set(scriptPathToRemotePath(p), { localPath: p, content }); + } + return byRemotePath; +} + export async function buildTracker(changes: Change[]) { const tracker: ChangeTracker = { scripts: [], @@ -5057,6 +5108,14 @@ export async function push( } if (autoRegenerate && tree) { + // Pass 1 only ever walks the change set, so anything imported through a + // module the push leaves alone is still a dead end here. + await resolvePlaceholdersFromLocal( + tree, + localScriptsByRemotePath(localMap), + opts.defaultTs, + ); + // Propagate staleness through imports + upload script content to // raw_script_temp so the dep job can resolve cross-folder relative imports // via temp_script_refs (instead of hitting 404s for not-yet-deployed @@ -5540,27 +5599,19 @@ export async function push( return; } - let permissionedAsContext: PermissionedAsContext | undefined = undefined; - if (parseSyncBehavior(opts.syncBehavior) >= 1) { - const user = await wmill.whoami({ workspace: workspace.workspaceId }); - const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + const permissionedAsContext: PermissionedAsContext | undefined = + await buildPermissionedAsContext( + workspace.workspaceId, + opts.syncBehavior, ); - permissionedAsContext = { - userCache: new Map(), - userIsAdminOrDeployer, - userEmail: user.email, - }; - + if (permissionedAsContext) { // ws_specific_flag changes have no content payload, so they don't // affect permissioned_as resolution — filter them out before the // pre-check (which expects only added/edited/deleted). await preCheckPermissionedAs( changes.filter((c) => c.name !== "ws_specific_flag"), - user.email, - userIsAdminOrDeployer, + permissionedAsContext.userEmail, + permissionedAsContext.userIsAdminOrDeployer, opts.acceptOverridingPermissionedAsWithSelf ?? false, !!process.stdin.isTTY, ); @@ -6204,7 +6255,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // Flow folder doesn't exist locally — delete on server @@ -6249,7 +6300,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // App folder doesn't exist locally — delete on server @@ -6295,7 +6346,11 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message, defaultTs: opts.defaultTs }, + { + message: opts.message, + defaultTs: opts.defaultTs, + permissionedAsContext, + }, ); } else { // The entire raw app folder was deleted locally, @@ -6607,6 +6662,22 @@ export async function push( ), ); } + // Both delete handlers move the item to the workspace trashbin first; without + // this the CLI is the only surface that never says so, and the deletion reads + // as final. + const deletedSecretBearing = changes.filter( + (c) => + c.name === "deleted" && + secretBearingObjectKind(c.path) !== undefined && + !failedChanges.some((f) => f.path === c.path), + ); + if (deletedSecretBearing.length > 0) { + log.info( + colors.gray( + `${describeSecretBearingChanges(deletedSecretBearing)} deleted. The workspace trashbin keeps a deleted item for three days; a workspace admin can restore it with \`wmill trash list\` and \`wmill trash restore \`, or from Workspace settings -> Trashbin.`, + ), + ); + } if (failedChanges.length > 0) { // Not process.exit: under Node a piped stdout write is async, so exiting // here would truncate the JSON result mid-object for CI consumers. diff --git a/cli/src/commands/trash/trash.ts b/cli/src/commands/trash/trash.ts new file mode 100644 index 0000000000..1d571e2884 --- /dev/null +++ b/cli/src/commands/trash/trash.ts @@ -0,0 +1,147 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { apiErrorMessage, formatTimestamp } from "../../utils/utils.ts"; + +async function list( + opts: GlobalOptions & { + json?: boolean; + kind?: string; + page?: number; + limit?: number; + } +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + if (opts.page !== undefined && opts.page < 1) { + throw new Error("--page starts at 1"); + } + + const items = await wmill.listTrash({ + workspace: workspace.workspaceId, + itemKind: opts.kind, + // The trash endpoint counts pages from 0, unlike the API's other list + // endpoints whose `page` starts at 1; the flag counts from 1 like those. + page: opts.page === undefined ? undefined : opts.page - 1, + perPage: opts.limit, + }); + + if (opts.json) { + console.log(JSON.stringify(items)); + return; + } + if (items.length === 0) { + log.info("No trashed items found."); + return; + } + new Table() + .header(["ID", "Kind", "Path", "Deleted by", "Deleted at", "Expires at"]) + .padding(2) + .border(true) + .body( + items.map((item) => [ + String(item.id), + item.item_kind, + item.item_path, + item.deleted_by, + formatTimestamp(item.deleted_at), + formatTimestamp(item.expires_at), + ]) + ) + .render(); + log.info( + colors.gray( + "`wmill trash get ` shows what an item held, `wmill trash restore ` puts it back." + ) + ); +} + +async function get(opts: GlobalOptions & { json?: boolean }, id: number) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const item = await wmill.getTrashItem({ + workspace: workspace.workspaceId, + id, + }); + + if (opts.json) { + console.log(JSON.stringify(item)); + return; + } + console.log(colors.bold("ID:") + " " + item.id); + console.log(colors.bold("Kind:") + " " + item.item_kind); + console.log(colors.bold("Path:") + " " + item.item_path); + console.log(colors.bold("Deleted by:") + " " + item.deleted_by); + console.log(colors.bold("Deleted at:") + " " + formatTimestamp(item.deleted_at)); + console.log(colors.bold("Expires at:") + " " + formatTimestamp(item.expires_at)); + console.log(colors.bold("Data:")); + console.log(JSON.stringify(item.item_data, null, 2)); +} + +async function restore(opts: GlobalOptions, ...ids: number[]) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + let failed = 0; + for (const id of ids) { + try { + const message = await wmill.restoreTrashItem({ + workspace: workspace.workspaceId, + id, + }); + log.info(colors.green(message)); + } catch (e) { + failed += 1; + log.error( + `Could not restore trash item ${id}: ${apiErrorMessage(e) ?? String(e)}` + ); + } + } + if (failed > 0) { + process.exitCode = 1; + } +} + +const command = new Command() + .description( + "List, inspect and restore items deleted in the last three days (requires admin)" + ) + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("list", "List trashed items, most recently deleted first") + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("get", "Show a trashed item and the data it was deleted with") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(get as any) + .command("restore", "Put trashed items back at their paths") + .arguments("") + .action(restore as any); + +export default command; diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 83b62f9eb4..4e6b0eeb16 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -23,7 +23,7 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { GlobalOptions, isSuperset, @@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; +import { readEffectiveSyncBehavior } from "../../core/conf.ts"; type Trigger = { http: HttpTrigger; @@ -222,9 +224,12 @@ export async function pushTrigger( } } +// `enabled` is operational state a sync deliberately does not carry: the server strips it from +// the workspace export and the push below never sends it, so a created trigger comes up enabled +// and pausing one stays a local decision. type NativeTriggerFile = Omit< NativeTrigger, - "external_id" | "workspace_id" | "error" + "external_id" | "workspace_id" | "error" | "enabled" >; export async function pushNativeTrigger( @@ -262,6 +267,7 @@ export async function pushNativeTrigger( service_config: result.service_config, error: result.error, summary: result.summary, + enabled: result.enabled, }; log.debug(`Native trigger ${serviceName}/${externalId} exists on remote`); } catch { @@ -620,8 +626,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise; } /** @@ -152,37 +154,42 @@ export async function pushVariable( log.debug(`Variable ${remotePath} does not exist on remote`); } + // extra_perms is synced independently via /acls/* (see applyExtraPermsDiff) + // so a perm-only edit never rewrites the variable value. Strip the field from + // the body that goes to update_variable / create_variable and treat it as a + // separate step both for the up-to-date short-circuit and after the write. + const { extra_perms: localPerms, ...localVariableBody } = localVariable; + if (variable) { - if (isSuperset(localVariable, variable)) { + if (isSuperset(localVariableBody, variable)) { log.debug(`Variable ${remotePath} is up-to-date`); - return; - } + } else { + log.debug(`Variable ${remotePath} is not up-to-date, updating`); - log.debug(`Variable ${remotePath} is not up-to-date, updating`); - - // Apply is_secret only when it differs from the remote (the value is always - // sent, so the server allows the flag change). Upgrades (non-secret->secret) - // always apply; downgrades only when explicitly allowed (single-file push) — - // see allowSecretDowngrade. `undefined` leaves the flag untouched. - let nextIsSecret: boolean | undefined = undefined; - if (localVariable.is_secret !== variable.is_secret) { - if (localVariable.is_secret) { - nextIsSecret = true; - } else if (allowSecretDowngrade) { - nextIsSecret = false; + // Apply is_secret only when it differs from the remote (the value is always + // sent, so the server allows the flag change). Upgrades (non-secret->secret) + // always apply; downgrades only when explicitly allowed (single-file push) — + // see allowSecretDowngrade. `undefined` leaves the flag untouched. + let nextIsSecret: boolean | undefined = undefined; + if (localVariableBody.is_secret !== variable.is_secret) { + if (localVariableBody.is_secret) { + nextIsSecret = true; + } else if (allowSecretDowngrade) { + nextIsSecret = false; + } } - } - await wmill.updateVariable({ - workspace, - path: remotePath.replaceAll(SEP, "/"), - alreadyEncrypted: !plainSecrets, - requestBody: { - ...localVariable, - is_secret: nextIsSecret, - ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), - }, - }); + await wmill.updateVariable({ + workspace, + path: remotePath.replaceAll(SEP, "/"), + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariableBody, + is_secret: nextIsSecret, + ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), + }, + }); + } } else { log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await wmill.createVariable({ @@ -190,11 +197,22 @@ export async function pushVariable( alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath.replaceAll(SEP, "/"), - ...localVariable, + ...localVariableBody, ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), }, }); } + + // Synced whether or not the body changed. No refetch: folder perms are never + // merged onto item.extra_perms, and the update/create body carries no + // extra_perms, so the value getVariable read above is still the remote one. + await applyExtraPermsDiff( + workspace, + "variable", + remotePath.replaceAll(SEP, "/"), + localPerms, + (variable as any)?.extra_perms, + ); } async function push( diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1fcce7a43f..0c746d2261 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null { } export const GLOBAL_CONFIG_OPT = { noCdToRoot: false }; -function findWmillYaml(): string | null { + +// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir. +function locateWmillYaml(): string | null { const startDir = resolve(process.cwd()); const isInGitRepo = isGitRepository(); const gitRoot = isInGitRepo ? getGitRepoRoot() : null; @@ -176,6 +178,13 @@ function findWmillYaml(): string | null { currentDir = parentDir; } + return foundPath; +} + +function findWmillYaml(): string | null { + const startDir = resolve(process.cwd()); + const foundPath = locateWmillYaml(); + // If wmill.yaml was found in a parent directory, warn the user and change working directory if ( !GLOBAL_CONFIG_OPT.noCdToRoot && @@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } +/** + * Look up one `workspaces` entry, for diagnostics only. readConfigFile() must + * not be used for that: it chdirs to the config's directory, exits on an + * unsupported syncBehavior and throws on a malformed file. A diagnostic may + * never fail or relocate the command it is diagnosing. + */ +export async function peekWorkspaceEntry( + workspaceName: string +): Promise { + if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) { + return undefined; + } + const wmillYamlPath = locateWmillYaml(); + if (!wmillYamlPath) { + return undefined; + } + try { + const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions; + const workspaces = + conf?.workspaces ?? + conf?.gitBranches ?? + conf?.environments ?? + conf?.git_branches; + const entry = (workspaces as any)?.[workspaceName]; + return typeof entry === "object" && entry !== null ? entry : undefined; + } catch (e) { + log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`); + return undefined; + } +} + let legacyConfigWarned = false; export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { @@ -630,6 +670,83 @@ export async function getEffectiveSettings( return effective; } +// Resolve workspace name from a --branch override (git branch → workspace name). +// Falls back to using the branch value as-is (backward compat: old key = branch name). +function resolveWsNameFromBranch(opts: SyncOptions, branchName: string): string { + const match = findWorkspaceByGitBranch(opts.workspaces, branchName); + return match ? match[0] : branchName; +} + +// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, +// then --workspace → matching config key (incl. when --base-url is set). Returns +// undefined when no flag-based resolution applies; callers then fall back to +// inferWsNameFromProfile on the resolved workspace profile. +export function resolveWsNameForConfigFromFlags( + opts: SyncOptions & { branch?: string; workspace?: string } +): string | undefined { + if (opts.branch) { + return resolveWsNameFromBranch(opts, opts.branch); + } + if (opts.workspace) { + // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, + // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. + const validKeys = getWorkspaceNames(opts.workspaces); + if (validKeys.includes(opts.workspace)) { + return opts.workspace; + } + } + return undefined; +} + +/** + * Match a workspace config entry to a resolved workspace profile by remote + + * workspace id. The fallback for when no flag names the entry outright. + */ +export function inferWsNameFromProfile( + opts: SyncOptions, + profile: { remote: string; workspaceId: string } +): string | undefined { + if (!opts.workspaces) return undefined; + for (const name of getWorkspaceNames(opts.workspaces)) { + const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; + if (!entry?.baseUrl) continue; + try { + const entryUrl = new URL(entry.baseUrl).toString(); + const profileUrl = new URL(profile.remote).toString(); + const entryWsId = entry.workspaceId ?? name; + if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { + return name; + } + } catch { + continue; + } + } + return undefined; +} + +/** + * `syncBehavior` as the workspace being pushed to sees it. The top level alone + * misses a `workspaces..overrides.syncBehavior`, which is where a repo + * that varies settings per workspace puts it, and the entry to read is the one + * `--workspace` names — falling back to the profile, then to the git branch — + * the same order `sync push` resolves it in. + */ +export async function readEffectiveSyncBehavior( + opts: { workspace?: string }, + profile?: { remote: string; workspaceId: string } +): Promise { + const config = await readConfigFile({ warnIfMissing: false }); + const named = resolveWsNameForConfigFromFlags({ ...config, ...opts }); + const effective = await getEffectiveSettings( + config, + undefined, + false, + true, + named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined) + ); + return effective.syncBehavior; +} + const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]); /** diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 35bd6bc2e7..8c01755492 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.804.0"; +export const VERSION = "1.812.0"; diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 502157f0a7..80b27ea52e 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -20,6 +20,7 @@ import { import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts"; import { readConfigFile, + peekWorkspaceEntry, findWorkspaceByGitBranch, getEffectiveWorkspaceId, getWmillYamlPath, @@ -219,6 +220,9 @@ async function tryResolveWorkspace( // First try: look up workspace by name in wmill.yaml workspaces config const config = await readConfigFile({ warnIfMissing: false }); const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined; + // What wmill.yaml said to target, kept for the fallback below: a profile + // found by name can silently point somewhere else entirely. + let configuredTarget: { workspaceId: string; baseUrl: string } | undefined; if (wsEntry?.baseUrl) { const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry); let normalizedBaseUrl: string; @@ -231,6 +235,8 @@ async function tryResolveWorkspace( }; } + configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl }; + // Find matching profile by baseUrl + workspaceId const allProfs = await allWorkspaces(opts.configDir); const matching = allProfs.filter( @@ -283,6 +289,22 @@ async function tryResolveWorkspace( ), }; } + if ( + configuredTarget && + (e.workspaceId !== configuredTarget.workspaceId || + e.remote !== configuredTarget.baseUrl) + ) { + log.warnStderr( + colors.yellow( + `⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` + + ` Run: wmill workspace add ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}` + ) + ); + } + log.infoStderr( + `Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}` + ); (opts as any).__secret_workspace = e; return { isError: false, value: e }; } @@ -486,6 +508,8 @@ export async function resolveWorkspace( return process.exit(-1); } + let resolved: Workspace | undefined; + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { let existingWorkspace = await getWorkspaceByName( @@ -523,19 +547,45 @@ export async function resolveWorkspace( ); return process.exit(-1); } - return { + resolved = { ...existingWorkspace, token: opts.token, }; } } - return { + resolved ??= { remote: normalizedBaseUrl, workspaceId: opts.workspace, name: opts.workspace, token: opts.token, }; + + // --base-url pins the target, so wmill.yaml's `workspaces` block is never + // consulted and `--workspace` reaches the API as a workspace id. Name the + // id being sent, and the mapping being skipped, before the request 404s + // on an id the user never typed. + // Only an explicit `workspaceId:` is worth reporting: an entry without one + // maps the name to itself, leaving nothing to correct. + const yamlEntry = await peekWorkspaceEntry(opts.workspace); + const yamlWorkspaceId = yamlEntry?.workspaceId; + if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) { + log.warnStderr( + colors.yellow( + `⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` + + ` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.` + ) + ); + } + log.infoStderr( + `Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` + + (resolved.name !== resolved.workspaceId + ? `, profile '${resolved.name}')` + : ")") + ); + (opts as any).__secret_workspace = resolved; + return resolved; } else { log.infoStderr( colors.red( diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 5ac48753d6..d0301373da 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,13 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { + extractFolderPath, + isAppFolderMetadataFile, + isRawAppFolderMetadataFile, +} from "../utils/resource_folders.ts"; +import { deploysWithRawApp } from "../utils/app_files.ts"; +import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { userCache: Map; @@ -10,6 +17,35 @@ export interface PermissionedAsContext { userEmail: string; } +/** + * The whole-tree `sync push` and the single-item `push` commands must resolve + * ownership the same way, so both build the context here: a push that leaves it + * undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it. + * Undefined below syncBehavior v1, where that reassignment is the contract, and + * for a caller who is neither admin nor in `wm_deployers` the backend enforces + * it anyway — the flag on the context is what keeps the CLI from claiming + * otherwise. + */ +export async function buildPermissionedAsContext( + workspace: string, + syncBehavior: string | number | undefined +): Promise { + if (parseSyncBehavior(syncBehavior) < 1) { + return undefined; + } + const user = await wmill.whoami({ workspace }); + const userIsAdminOrDeployer = + user.is_admin || (user.groups ?? []).includes("wm_deployers"); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + ); + return { + userCache: new Map(), + userIsAdminOrDeployer, + userEmail: user.email, + }; +} + async function ensureUserCache( workspace: string, cache: Map @@ -59,6 +95,45 @@ function contentHasOnBehalfOf(content: string, typeStr: string): boolean { return false; } +type AppTypeStr = "app" | "raw_app"; + +function isAppTypeStr(typeStr: string): typeStr is AppTypeStr { + return typeStr === "app" || typeStr === "raw_app"; +} + +/** The app folder a file belongs to. `isAppFolderMetadataFile` and its raw twin + * match a literal `/`, unlike `extractFolderPath` — so normalize before either, + * or a Windows path takes a different branch from the same file on Linux. */ +function appFolderOf(path: string, typeStr: AppTypeStr): string { + return extractFolderPath(path, typeStr) ?? path; +} + +function toPosix(path: string): string { + return path.replaceAll("\\", "/"); +} + +/** App folders whose own metadata file is being added or deleted, which is how a + * whole app arrives or goes rather than being redeployed. Neither takes an owner + * over: a create has none yet, and a delete leaves none behind. */ +function appsArrivingOrLeaving(changes: Change[]): Set { + const folders = new Set(); + for (const change of changes) { + if (change.name === "edited") continue; + const path = toPosix(change.path); + if (!isAppFolderMetadataFile(path) && !isRawAppFolderMetadataFile(path)) { + continue; + } + let typeStr: string; + try { + typeStr = getTypeStrFromPath(path); + } catch { + continue; + } + if (isAppTypeStr(typeStr)) folders.add(appFolderOf(path, typeStr)); + } + return folders; +} + export async function preCheckPermissionedAs( changes: Change[], userEmail: string, @@ -71,6 +146,12 @@ export async function preCheckPermissionedAs( if (userIsAdminOrDeployer) return; const wouldChangeItems: { path: string; currentOwner: string }[] = []; + const addItem = (item: { path: string; currentOwner: string }) => { + if (!wouldChangeItems.some((i) => i.path === item.path)) { + wouldChangeItems.push(item); + } + }; + const arrivingOrLeaving = appsArrivingOrLeaving(changes); for (const change of changes) { let typeStr: string; @@ -80,6 +161,22 @@ export async function preCheckPermissionedAs( continue; } + // An app is redeployed whole by any change to any of the files it actually + // sends — added, edited or deleted alike — so its policy is rewritten + // regardless of what the file holds. Settled here, before the content the + // other kinds parse to find their owner, which an app has none of to parse. + if (isAppTypeStr(typeStr)) { + const path = toPosix(change.path); + const folder = appFolderOf(path, typeStr); + if ( + !arrivingOrLeaving.has(folder) && + (typeStr === "app" || deploysWithRawApp(path.slice(folder.length))) + ) { + addItem({ path: folder, currentOwner: "(app policy owner)" }); + } + continue; + } + if (change.name === "added") { const content = change.content; if (!content) continue; @@ -100,11 +197,6 @@ export async function preCheckPermissionedAs( const label = typeStr === "script" ? "(script owner)" : "(flow owner)"; wouldChangeItems.push({ path: change.path, currentOwner: label }); - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); } continue; } @@ -147,12 +239,6 @@ export async function preCheckPermissionedAs( } } continue; - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); - continue; } else if (typeStr === "schedule") { const match = beforeContent.match( /email:\s*["']?([^\s"']+)["']?/ diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 66fc77d230..4c95090bcf 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -214,9 +214,15 @@ export async function pushWorkspaceSettings( } // Exclude fields that are never applied here: slack_team_id/slack_name are OAuth-only, - // and name is not applied on pull (see below), so a name-only diff stays a no-op. + // and name is never applied (see below), so a name-only diff stays a no-op. color is + // applied only when the file carries it, so an unset one leaves the comparison too. const { slack_team_id: _lst, slack_name: _lsn, name: _ln, ...comparableLocal } = localSettings; const { slack_team_id: _rst, slack_name: _rsn, name: _rn, ...comparableRemote } = settings; + const colorManaged = localSettings.color != null; + if (!colorManaged) { + delete comparableLocal.color; + delete comparableRemote.color; + } if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; @@ -351,10 +357,9 @@ export async function pushWorkspaceSettings( }); } - // Workspace display name is intentionally not applied on pull: settings.yaml is shared - // across a repo's branches, so applying it would let one workspace's name overwrite - // another's when both sync the same repo. It stays in the file (written on push), but a - // live workspace is only renamed by its owner. + // Workspace display name is intentionally never applied by `sync push`: settings.yaml is + // shared across a repo's branches, so applying it would let one workspace's name overwrite + // another's when both sync the same repo. `sync pull` still records it. if (localSettings.mute_critical_alerts != settings.mute_critical_alerts) { log.debug(`Updating mute critical alerts...`); @@ -366,7 +371,9 @@ export async function pushWorkspaceSettings( }); } - if (localSettings.color != settings.color) { + // A color is applied only when the file carries one: `sync pull` omits the key for a + // workspace without a color, so an unset key means "not managed by git", never "clear". + if (colorManaged && localSettings.color != settings.color) { log.debug(`Updating workspace color...`); await wmill.changeWorkspaceColor({ workspace, diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 2a9bb34fe0..202a6feaad 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -165,6 +165,8 @@ No CI workflow runs \`wmill sync push\` automatically, so deploy directly from t - \`wmill sync push --dry-run\` to preview. - \`wmill sync push\` to apply. +A push deletes remote items that have no local file. They land in the workspace trashbin for three days: \`wmill trash list\` shows them and \`wmill trash restore \` puts one back (both need a workspace admin). + ### In both cases Only deploy when the user explicitly asks to deploy, publish, push, or ship — not when they say "run", "try", or "test". For testing local edits use the per-entity \`preview\` commands (\`wmill script preview\`, \`wmill flow preview\`) — they don't deploy. diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 5271167a07..f58092fdac 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1037,12 +1037,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -1824,12 +1828,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -2705,12 +2713,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -4580,6 +4592,30 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -4587,10 +4623,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -4598,10 +4639,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -4609,7 +4652,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -4650,13 +4693,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # @@ -5535,7 +5582,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments/enabled_tools).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -5868,6 +5915,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. + +\`\`\`tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +\`\`\` + +\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -5960,8 +6020,8 @@ export async function main(user_id: string) { const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); // Insert/Update - await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; - await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; + await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`.execute(); + await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`.execute(); return user; } @@ -5980,8 +6040,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user \`\`\` @@ -5990,12 +6050,14 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — \`get_user\`, not \`a\`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. -6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +3. **Terminate every datatable statement** — the tagged template and \`db.query(...)\` only build a statement. It runs when you call \`fetch\` / \`fetchOne\` / \`fetchOneScalar\` / \`execute\` (\`fetch\` / \`fetch_one\` / \`fetch_one_scalar\` / \`execute\` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — \`get_user\`, not \`a\`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. +7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers @@ -6657,14 +6719,50 @@ A caught failure reads the same whether it came from a task or from a \`step()\` Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\` \`\`\`typescript +/** + * Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (\`fetch\`, \`fetch#2\`, \`fetch#3\`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and \`attempts\`. Retries with no + * \`delay\` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: \`2\` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a \`multiplier\` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A \`taskScript\` + * target is keyed on the arguments it is called with. It has no effect on a + * \`taskFlow\` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; } /** @@ -6682,9 +6780,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -6740,12 +6840,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -6828,6 +6932,30 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -6835,10 +6963,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -6846,10 +6979,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -6857,7 +6992,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -6898,13 +7033,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one \`\`wait_for_approval\`\` step. # @@ -7326,7 +7465,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory +Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file **Arguments:** \`[directory:string]\` @@ -7619,6 +7758,27 @@ Manage API tokens - \`--expiration \` - Token expiration (ISO 8601 timestamp) - \`token delete \` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) +- \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- \`--limit \` - Number of items to return (default 100, max 1000) +- \`--page \` - Page to return, starting at 1 + +**Subcommands:** + +- \`trash list\` - List trashed items, most recently deleted first + - \`--json\` - Output as JSON (for piping to jq) + - \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - \`--limit \` - Number of items to return (default 100, max 1000) + - \`--page \` - Page to return, starting at 1 +- \`trash get \` - Show a trashed item and the data it was deleted with + - \`--json\` - Output as JSON (for piping to jq) +- \`trash restore \` - Put trashed items back at their paths + ### trigger trigger related commands @@ -8424,6 +8584,21 @@ properties: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + items: + type: string + description: 'Origins allowed to call this route cross-origin, matched against + the request''s Origin header (ignoring case) and echoed back on a match. When + set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin + the runnable returns via wm_headers. Use [''*''] to opt out of any restriction, + including the http_route_default_allowed_origins instance setting. An empty + list is not a configuration and resolves exactly as null does. When null, the + instance setting applies, or Access-Control-Allow-Origin: * if it is unset. + Ignored on a static website, which has no authentication of its own and so hands + out public files: restricting which browsers may read them protects nothing + while breaking cross-origin webfonts and fetches. A single-file static asset + is not exempt, since it can carry an authentication_method.' error_handler_path: type: string description: Path to a script to run when the triggered job fails. A bare path, diff --git a/cli/src/main.ts b/cli/src/main.ts index ccce20da30..fa894b6f2d 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -29,7 +29,7 @@ import lint from "./commands/lint/lint.ts"; import dev from "./commands/dev/dev.ts"; import { GlobalOptions } from "./types.ts"; import { OpenAPI } from "../gen/index.ts"; -import { getHeaders } from "./utils/utils.ts"; +import { apiErrorMessage, getHeaders } from "./utils/utils.ts"; import { detectAuthGatewayChallenge } from "./utils/http_guards.ts"; import { setShowDiffs } from "./core/conf.ts"; import { markRequestsAsCliClient } from "./core/client.ts"; @@ -48,6 +48,7 @@ import job from "./commands/job/job.ts"; import group from "./commands/group/group.ts"; import audit from "./commands/audit/audit.ts"; import token from "./commands/token/token.ts"; +import trash from "./commands/trash/trash.ts"; import generateMetadata from "./commands/generate-metadata/generate-metadata.ts"; import docs from "./commands/docs/docs.ts"; import config from "./commands/config/config.ts"; @@ -214,6 +215,7 @@ const command = new Command() .command("group", group) .command("audit", audit) .command("token", token) + .command("trash", trash) .command("generate-metadata", generateMetadata) .command("docs", docs) .command("config", config) @@ -321,14 +323,9 @@ async function main() { await command.parse(args); } catch (e) { - if (e && typeof e === "object" && "name" in e && e.name === "ApiError") { - const body = (e as any).body; - let bodyStr = typeof body === "object" && body !== null ? JSON.stringify(body) : String(body ?? ""); - // Strip backend source file references like (flows.rs:1400) or @scripts.rs:123:45 - bodyStr = bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); - log.error( - "Server failed. " + (e as any).statusText + ": " + bodyStr - ); + const apiError = apiErrorMessage(e); + if (apiError !== undefined) { + log.error("Server failed. " + apiError); } else if (e instanceof Error) { log.error(e.message); } else if (e !== undefined && e !== null) { diff --git a/cli/src/types.ts b/cli/src/types.ts index 9c517ff3c7..5713d06390 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -234,7 +234,7 @@ export async function pushObj( if (!rawAppName) { throw new Error(`Could not extract raw app name from path: ${p}`); } - await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs); + await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs, permissionedAsContext); } else if (typeEnding === "folder") { await pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { diff --git a/cli/src/utils/app_files.ts b/cli/src/utils/app_files.ts new file mode 100644 index 0000000000..ca1be1de0e --- /dev/null +++ b/cli/src/utils/app_files.ts @@ -0,0 +1,48 @@ +import { + APP_BACKEND_FOLDER, + RECORDINGS_FOLDER, +} from "../commands/app/app_metadata.ts"; + +/** Directories under a raw app that no push sends. */ +const NEVER_DEPLOYED_DIRS = new Set([ + "node_modules", + "dist", + ".claude", + "sql_to_apply", +]); + +/** Files under a raw app that no push sends. */ +const NEVER_DEPLOYED_FILES = new Set([ + "package-lock.json", + "DATATABLES.md", + "AGENTS.md", + "wmill.d.ts", +]); + +/** + * Whether an app-root-relative path (`/` separators, leading slash optional) + * reaches the server through any of a push's three channels: `raw_app.yaml` as + * metadata, the backend folder as runnables, the rest bundled by + * `collectAppFiles`. A path this rejects deploys nothing, so changing it is not + * a change to the app however much the sync diff lists it. `collectAppFiles` + * must not drift from this — it reads the same two sets. + */ +export function deploysWithRawApp(relativePath: string): boolean { + const segments = relativePath.split("/").filter(Boolean); + if (segments.length === 0) return false; + const name = segments[segments.length - 1]; + const dirs = segments.slice(0, -1); + // The sets below describe the bundle, which never walks into the backend + // folder — applying them there would strip a runnable whose file shares a + // name (`backend/wmill.d.ts` is the runnable `wmill.d`). Depth 1 because + // `loadRunnablesFromBackend` reads that folder's top level only. + if (dirs[0] === APP_BACKEND_FOLDER) return dirs.length === 1; + if (NEVER_DEPLOYED_FILES.has(name)) return false; + if (dirs.some((d) => NEVER_DEPLOYED_DIRS.has(d))) return false; + // Session recordings are written at the app root only, so an app with a + // `recordings/` component folder of its own still ships it. + if (dirs[0] === RECORDINGS_FOLDER) return false; + return true; +} + +export { NEVER_DEPLOYED_DIRS, NEVER_DEPLOYED_FILES }; diff --git a/cli/src/utils/dependency_tree.ts b/cli/src/utils/dependency_tree.ts index b0b75e659e..959a77b478 100644 --- a/cli/src/utils/dependency_tree.ts +++ b/cli/src/utils/dependency_tree.ts @@ -5,7 +5,7 @@ import { Workspace } from "../commands/workspace/workspace.ts"; import * as wmill from "../../gen/services.gen.ts"; import type { ScriptLang } from "../../gen/types.gen.ts"; -import { ScriptLanguage } from "./script_common.ts"; +import { ScriptLanguage, inferContentTypeFromFilePath } from "./script_common.ts"; import { filterWorkspaceDependencies, generateScriptHash, @@ -14,6 +14,65 @@ import { updateMetadataGlobalLock, } from "./metadata.ts"; import { generateHash } from "./utils.ts"; +import { extractRelativeImports } from "./relative_imports.ts"; + +/** A local script file, keyed in `LocalScripts` by its Windmill remote path. */ +export interface LocalScriptSource { + localPath: string; + content: string; +} +export type LocalScripts = Map; + +/** + * Give every import target that is only a placeholder its local content and its + * own imports, so the graph continues through it. A tree seeded from a subset of + * the checkout otherwise dead-ends at any module outside that subset — a + * re-export barrel needing no edit, typically — hiding what it re-exports from + * `getTempScriptRefs`, which then resolves it against the deployed copy. + */ +export async function resolvePlaceholdersFromLocal( + tree: DoubleLinkedDependencyTree, + localScripts: LocalScripts, + defaultTs: "bun" | "deno" | undefined +): Promise { + // A module resolved here can expose placeholders of its own (a barrel behind + // a barrel), so keep going until a round resolves nothing. + for (;;) { + let resolved = false; + for (const remotePath of tree.placeholderPaths()) { + const local = localScripts.get(remotePath); + if (!local) continue; + let language: ScriptLanguage; + try { + language = inferContentTypeFromFilePath(local.localPath, defaultTs); + } catch { + // A bare `.sql` names no dialect, so its imports cannot be read here. + continue; + } + const imports = await extractRelativeImports( + local.content, + remotePath, + language + ); + // Never directly stale: it is outside the change set, so nothing relocks + // it. It is here to carry edges, and to be uploaded if it differs from + // what is deployed. + await tree.addNode( + remotePath, + local.content, + language, + "", + imports, + "script", + remotePath, + local.localPath, + false + ); + resolved = true; + } + if (!resolved) break; + } +} /** * Diff local scripts against deployed versions, upload only those that differ. @@ -97,6 +156,9 @@ interface DependencyNode { originalPath: string; // Original path passed to handler (with extension for scripts) isRawApp?: boolean; // Only set for apps isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale) + // True while the node exists only because something imports it, so it carries + // no content and no imports of its own. + isPlaceholder: boolean; } export class DoubleLinkedDependencyTree { @@ -130,9 +192,11 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } const node = this.nodes.get(path)!; + node.isPlaceholder = false; node.content = content; node.stalenessHash = stalenessHash; node.language = language; @@ -155,7 +219,7 @@ export class DoubleLinkedDependencyTree { stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "dependencies", folder: "", originalPath: depsPath, - isDirectlyStale: !isUpToDate, + isDirectlyStale: !isUpToDate, isPlaceholder: false, }); } } @@ -169,6 +233,7 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } this.nodes.get(importPath)!.importedBy.add(path); @@ -309,6 +374,18 @@ export class DoubleLinkedDependencyTree { return this.nodes.keys(); } + /** + * Paths that exist only as somebody's import target, so the traversal stops + * at them instead of continuing into what they themselves import. + */ + placeholderPaths(): string[] { + const result: string[] = []; + for (const [path, node] of this.nodes.entries()) { + if (node.isPlaceholder) result.push(path); + } + return result; + } + /** * Returns paths of all stale nodes (those with a staleReason). */ diff --git a/cli/src/utils/git.ts b/cli/src/utils/git.ts index 5ca06f427a..354e15c36e 100644 --- a/cli/src/utils/git.ts +++ b/cli/src/utils/git.ts @@ -1,6 +1,7 @@ import * as log from "../core/log.ts"; import { execSync, spawnSync } from "node:child_process"; import { WM_FORK_PREFIX } from "../core/constants.ts"; +import { SHARED_LOCK_DIR } from "./script_common.ts"; // Fork *workspace id* prefix ("wm-fork-"). WM_FORK_PREFIX is the *branch* // prefix ("wm-fork") used inside the wm-fork// branch name. @@ -584,6 +585,11 @@ export function gitSyncDeployPush(params: { git(["add", "wmill-lock.yaml", `${parent_path}**`], { allowFail: true }); } } + // A shared lockfile (`dedupeLockfiles`) lives under `locks/`, outside every + // item's path glob, and the pull rewrites it when a deployed script's lock + // changed. `-A` also stages the deletion of a swept one; the add fails only + // when nothing under `locks/` exists or is tracked. + git(["add", "-A", "--", SHARED_LOCK_DIR], { allowFail: true }); // `git diff --cached --quiet` exits 1 iff there is something staged. const staged = git(["diff", "--cached", "--quiet"], { allowFail: true }); diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 489a877af7..0801c03911 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -353,6 +353,23 @@ export function formatTimestamp(ts: string): string { return new Date(ts).toISOString().replace("T", " ").substring(0, 19); } +/** + * ": " for an error thrown by the generated API client, + * undefined for anything else. Backend source references such as + * `(flows.rs:1400)` are stripped from the body. + */ +export function apiErrorMessage(e: unknown): string | undefined { + if (!(e && typeof e === "object" && "name" in e && e.name === "ApiError")) { + return undefined; + } + const { body, statusText } = e as { body?: unknown; statusText?: string }; + const bodyStr = + typeof body === "object" && body !== null + ? JSON.stringify(body) + : String(body ?? ""); + return statusText + ": " + bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); +} + /** * Validate that required arguments are present when no -d data was provided. * Fetches the schema from the API and checks required fields. diff --git a/cli/test/app_access_mode_unit.test.ts b/cli/test/app_access_mode_unit.test.ts index 12185d6d99..42a3fd082a 100644 --- a/cli/test/app_access_mode_unit.test.ts +++ b/cli/test/app_access_mode_unit.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import { - executionModeFromAppFile, + executionModeForPush, generatingPolicy, markAccessFromPolicy, } from "../src/commands/app/app.ts"; @@ -13,16 +13,39 @@ test("the access mode survives the app.yaml round trip", async () => { guest.policy = undefined; expect(guest.guests).toBe(true); expect(guest.public).toBeUndefined(); - expect(executionModeFromAppFile(guest)).toBe("guest"); - await generatingPolicy(guest, "u/test/app", executionModeFromAppFile(guest)); + expect(executionModeForPush(guest, undefined)).toBe("guest"); + await generatingPolicy( + guest, + "u/test/app", + executionModeForPush(guest, undefined), + undefined, + ); expect(guest.policy.execution_mode).toBe("guest"); const anonymous: any = { policy: { execution_mode: "anonymous" }, value: {} }; markAccessFromPolicy(anonymous); anonymous.policy = undefined; expect(anonymous.public).toBe(true); - expect(executionModeFromAppFile(anonymous)).toBe("anonymous"); + expect(executionModeForPush(anonymous, undefined)).toBe("anonymous"); - expect(executionModeFromAppFile({ policy: { execution_mode: "publisher" } })).toBe("publisher"); - expect(executionModeFromAppFile({})).toBe("publisher"); + expect(executionModeForPush({ policy: { execution_mode: "publisher" } }, undefined)).toBe("publisher"); + expect(executionModeForPush({}, undefined)).toBe("publisher"); +}); + +// `viewer` is the narrowest mode — each runnable runs as the viewer, not as the +// app's identity — and the only one with no marker in the file, so both ways it +// can reach a push must survive rather than widen to `publisher`. +test("viewer is never widened to publisher by a push", () => { + // Carried over from the deployed app: a pull writes no marker for it. + expect(executionModeForPush({}, { execution_mode: "viewer" })).toBe("viewer"); + // Stated by the file, which is all a first push has to go on. + expect(executionModeForPush({ policy: { execution_mode: "viewer" } }, undefined)).toBe("viewer"); + // The open-access markers still win, in either direction. + expect(executionModeForPush({ public: true }, { execution_mode: "viewer" })).toBe("anonymous"); + expect(executionModeForPush({}, { execution_mode: "anonymous" })).toBe("publisher"); + // A stated mode is authoritative both ways: the carry-over is for a file that + // says nothing, so it must not pin a deployed app to `viewer` forever. + expect( + executionModeForPush({ policy: { execution_mode: "publisher" } }, { execution_mode: "viewer" }) + ).toBe("publisher"); }); diff --git a/cli/test/base_url_workspace_resolution_unit.test.ts b/cli/test/base_url_workspace_resolution_unit.test.ts new file mode 100644 index 0000000000..66f618d113 --- /dev/null +++ b/cli/test/base_url_workspace_resolution_unit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { resolveWorkspace } from "../src/core/context.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; + +const BASE_URL = "http://localhost:9999/"; + +// --base-url pins the target: --workspace reaches the API as a workspace id and +// wmill.yaml is not consulted. The warning that says so may only peek at the +// file — readConfigFile() exits on an unsupported syncBehavior and throws on a +// malformed one, so resolving through it lets an unrelated config fail a +// command that never needed it. +async function withWmillYaml( + wmillYaml: string, + fn: (opts: GlobalOptions) => Promise +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_")); + const originalCwd = process.cwd(); + try { + await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml); + await writeFile(await getWorkspaceConfigFilePath(configDir), ""); + + process.chdir(repoDir); + await fn({ + configDir, + baseUrl: BASE_URL, + token: "sometoken", + workspace: "staging", + } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +describe("--base-url workspace resolution", () => { + const rejectedConfigs: [string, string][] = [ + ["an unsupported syncBehavior", "syncBehavior: v2\n"], + ["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'], + ]; + + for (const [label, wmillYaml] of rejectedConfigs) { + test(`resolves despite ${label}`, async () => { + await withWmillYaml(wmillYaml, async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + }); + }); + } + + test("a workspaces mapping never overrides the explicit workspace id", async () => { + await withWmillYaml( + "workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n", + async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + } + ); + }); +}); diff --git a/cli/test/dependency_tree_unit.test.ts b/cli/test/dependency_tree_unit.test.ts index 1a92bb9894..8e9c733c00 100644 --- a/cli/test/dependency_tree_unit.test.ts +++ b/cli/test/dependency_tree_unit.test.ts @@ -2,7 +2,10 @@ import { expect, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import * as path from "node:path"; -import { DoubleLinkedDependencyTree } from "../src/utils/dependency_tree.ts"; +import { + DoubleLinkedDependencyTree, + resolvePlaceholdersFromLocal, +} from "../src/utils/dependency_tree.ts"; // addNode consults wmill-lock.yaml from cwd for workspace deps; run inside a // temp dir so the test never reads/writes the repo's own lock file. @@ -104,3 +107,62 @@ test("getAllTempScriptRefs is a superset of getTempScriptRefs for any node", asy }); }); }); + +// Two barrels deep so the fixpoint matters: resolving the first one is what +// puts the second in the tree, and only a further round reaches the leaf. +test("resolvePlaceholdersFromLocal walks the graph through unresolved barrels", async () => { + await withTempDir(async () => { + const tree = new DoubleLinkedDependencyTree(); + await tree.addNode( + "f/app/consumer", + `import { subtract } from "../barrel/index.ts"`, + "bun", + "", + ["f/barrel/index"], + "script", + "f/app/consumer", + "f/app/consumer.ts", + true, + ); + await tree.addNode( + "f/barrel/helper", + "export function subtract(a: number, b: number) { return a - b }", + "bun", + "", + [], + "script", + "f/barrel/helper", + "f/barrel/helper.ts", + true, + ); + // Neither barrel is in the change set, so both are bare import targets. + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({}); + + await resolvePlaceholdersFromLocal( + tree, + new Map([ + [ + "f/barrel/index", + { + localPath: "f/barrel/index.ts", + content: `export * from "./mid.ts"`, + }, + ], + [ + "f/barrel/mid", + { + localPath: "f/barrel/mid.ts", + content: `export * from "./helper.ts"`, + }, + ], + ]), + "bun", + ); + + // Only the leaf diverged from deployed, so only it was uploaded. + tree.setContentHash("f/barrel/helper", "hash_helper"); + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({ + "f/barrel/helper": "hash_helper", + }); + }); +}); diff --git a/cli/test/deploy_on_behalf_of_unit.test.ts b/cli/test/deploy_on_behalf_of_unit.test.ts index 2214c0dc37..7e21b24832 100644 --- a/cli/test/deploy_on_behalf_of_unit.test.ts +++ b/cli/test/deploy_on_behalf_of_unit.test.ts @@ -1,11 +1,12 @@ import { expect, test } from "bun:test"; import { deployItem } from "../windmill-utils-internal/src/deploy.ts"; -// `deployItem` spreads the source item into the request body, and a script's/flow's -// on_behalf_of names a username that only exists in the source -// workspace. Sending it to the target pairs one workspace's principal with the other's -// email, which the backend rejects. Deleting the spread is an easy regression, so pin -// that the key never reaches the wire. +// `deployItem` spreads the source item into the request body, and the principal it carries +// (`on_behalf_of`, at the top level for a script or flow and inside the policy for an app) +// names a username that only exists in the source workspace. Sending it to the target pairs +// one workspace's principal with the other's email, which the backend rejects. Deleting the +// spread is an easy regression, so pin that the principal never reaches the wire while the +// caller's chosen address does. function recordingProvider(captured: [string, any][], flowExists: boolean) { const source = { on_behalf_of_email: "alice@corp", @@ -32,6 +33,19 @@ function recordingProvider(captured: [string, any][], flowExists: boolean) { }), createScript: async (p: any) => void captured.push(["createScript", p.requestBody]), + existsApp: async () => false, + getAppByPath: async () => ({ + path: "f/x/a", + summary: "", + value: {}, + raw_app: false, + policy: { + execution_mode: "publisher", + on_behalf_of: "u/alice", + on_behalf_of_email: "alice@corp", + }, + }), + createApp: async (p: any) => void captured.push(["createApp", p.requestBody]), } as any; } @@ -65,19 +79,29 @@ test("deployItem: never sends the source workspace's on_behalf_of", async () => "dst", "alice@corp", ); + await deployItem( + recordingProvider(captured, false), + "app" as any, + "f/x/a", + "src", + "dst", + "alice@corp", + ); expect(captured.map(([fn]) => fn)).toEqual([ "createFlow", "updateFlow", "createScript", + "createApp", ]); - for (const [, body] of captured) { - // The email is still overridden with the caller's choice... - expect(body.on_behalf_of_email).toBe("alice@corp"); + for (const [name, body] of captured) { expect(body.preserve_on_behalf_of).toBe(true); + // Both surfaces spell it `on_behalf_of`; only its nesting differs — an app carries the + // identity inside its policy, the others at the top level. + const identity = name === "createApp" ? body.policy : body; + // The email is still overridden with the caller's choice... + expect(identity.on_behalf_of_email).toBe("alice@corp"); // ...while the principal is dropped, so the backend derives the target's own. - expect( - "on_behalf_of" in JSON.parse(JSON.stringify(body)), - ).toBe(false); + expect("on_behalf_of" in JSON.parse(JSON.stringify(identity))).toBe(false); } }); diff --git a/cli/test/gitsync_deploy_push_unit.test.ts b/cli/test/gitsync_deploy_push_unit.test.ts new file mode 100644 index 0000000000..f87e86bcc3 --- /dev/null +++ b/cli/test/gitsync_deploy_push_unit.test.ts @@ -0,0 +1,98 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gitSyncDeployPush } from "../src/utils/git.ts"; + +function git(cwd: string, ...args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +// A seeded clone of a bare remote, with `files` committed on main. +async function seededClone( + files: Record, +): Promise<{ bare: string; work: string }> { + const bare = await mkdtemp(join(tmpdir(), "wmill_deploy_push_bare_")); + execFileSync("git", ["init", "--quiet", "--bare", "--initial-branch=main", bare]); + const work = await mkdtemp(join(tmpdir(), "wmill_deploy_push_work_")); + git(work, "init", "--quiet", "--initial-branch=main"); + git(work, "config", "user.email", "seed@windmill.dev"); + git(work, "config", "user.name", "seed"); + for (const [path, content] of Object.entries(files)) { + await mkdir(join(work, path, ".."), { recursive: true }); + await writeFile(join(work, path), content); + } + git(work, "add", "-A"); + git(work, "commit", "--quiet", "-m", "seed"); + git(work, "remote", "add", "origin", `file://${bare}`); + git(work, "push", "--quiet", "-u", "origin", "main"); + return { bare, work }; +} + +function deployPushIn(work: string, path: string) { + const cwd = process.cwd(); + process.chdir(work); + try { + return gitSyncDeployPush({ + items: [{ path_type: "script", path, commit_msg: `deploy ${path}` }], + authorName: "windmill", + authorEmail: "windmill@windmill.dev", + }); + } finally { + process.chdir(cwd); + } +} + +test("a rewritten shared lockfile is committed with the deployed item", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline locks/requirements.in.lock'\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // What the deploy callback's pull leaves behind after `f/dd/a` was relocked: + // the item's own files are unchanged, only the shared file moved. + await writeFile(join(work, "locks/requirements.in.lock"), "requests==2.32.3\n"); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(work, "show", "--name-only", "--format=", "HEAD")).toBe( + "locks/requirements.in.lock", + ); + expect(git(bare, "cat-file", "-p", "main:locks/requirements.in.lock")).toBe( + "requests==2.32.3", + ); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a swept shared lockfile is committed as a deletion", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // The pull removed the last shared lockfile, and `locks/` with it. + await rm(join(work, "locks"), { recursive: true, force: true }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(bare, "ls-tree", "--name-only", "main", "locks/")).toBe(""); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a repository without shared lockfiles is left alone", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(false); + expect(git(bare, "rev-parse", "main")).toBe(git(work, "rev-parse", "HEAD")); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); diff --git a/cli/test/lint_orphan_metadata_unit.test.ts b/cli/test/lint_orphan_metadata_unit.test.ts new file mode 100644 index 0000000000..7066d10dca --- /dev/null +++ b/cli/test/lint_orphan_metadata_unit.test.ts @@ -0,0 +1,166 @@ +import { expect, test, describe } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import * as path from "node:path"; +import { runLint } from "../src/commands/lint/lint.ts"; + +const WMILL_YAML = "defaultTs: bun\nincludes:\n - f/**\nexcludes: []\n"; +const METADATA = "summary: test\nlock: ''\nschema:\n properties: {}\n"; + +async function write(dir: string, rel: string, content: string) { + const full = path.join(dir, rel); + await mkdir(path.dirname(full), { recursive: true }); + await writeFile(full, content, "utf-8"); +} + +/** + * Runs `fn` with a sync root at `/`, from which lint resolves + * every walked path. The name is a parameter because it is load-bearing: the + * folder suffixes lint classifies by (`.app`, `__mod`, …) are matched anywhere + * in a path, so a root carrying one must not change what lint reports. + */ +async function withSyncRoot( + rootName: string, + fn: (syncRoot: string) => Promise, + opts: { runFromParent?: boolean } = {}, +): Promise { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "wmill_lint_orphan_")); + const syncRoot = path.join(tempDir, rootName); + const originalCwd = process.cwd(); + try { + await write(syncRoot, "wmill.yaml", WMILL_YAML); + process.chdir(opts.runFromParent ? tempDir : syncRoot); + await fn(syncRoot); + } finally { + process.chdir(originalCwd); + await rm(tempDir, { recursive: true }); + } +} + +describe("orphan script metadata", () => { + test("reports metadata with no content file, with locks not required", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/paired.py", "def main():\n pass\n"); + await write(syncRoot, "f/paired.script.yaml", METADATA); + await write(syncRoot, "f/orphan.script.yaml", METADATA); + await write(syncRoot, "f/orphan_json.script.json", "{}\n"); + await write(syncRoot, "f/orphan_lock.script.lock", "some-dep==1.0.0\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.exitCode).toBe(1); + expect(report.issues.map((i) => i.path).sort()).toEqual([ + "f/orphan.script.yaml", + "f/orphan_json.script.json", + "f/orphan_lock.script.lock", + ]); + expect(report.issues[0].target).toBe("script"); + expect(report.issues[0].errors[0]).toContain("No script file found next to"); + }); + }); + + test("reports a module folder's own metadata with no content file", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/orphan__mod/script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual([ + "f/orphan__mod/script.yaml", + ]); + + // Linting the module folder itself: the walked paths no longer carry the + // `__mod/` boundary that says this is a module's metadata. + const inFolder = await runLint( + {} as any, + path.join(syncRoot, "f/orphan__mod"), + ); + + expect(inFolder.issues.map((i) => i.path)).toEqual(["script.yaml"]); + }); + }); + + test("reports orphans under a sync root named like a resource folder", async () => { + await withSyncRoot("acme.app", async (syncRoot) => { + await write(syncRoot, "f/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual(["f/orphan.script.yaml"]); + }); + }); + + test("does not report a paired module under a sync root named like a module folder", async () => { + // Run from OUTSIDE the checkout, the one invocation whose paths carry the + // root's own name: nothing above the sync root may be classified. + await withSyncRoot( + "repo__mod", + async (syncRoot) => { + await write(syncRoot, "f/example__mod/script.yaml", METADATA); + await write( + syncRoot, + "f/example__mod/script.ts", + "export function main() {}\n", + ); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("does not report a non-dotted folder resource's child", async () => { + // The dotted/non-dotted setting is read from the invocation directory, so + // an explicit target configured the other way must still be recognized. + await withSyncRoot( + "repo", + async (syncRoot) => { + await write(syncRoot, "f/a__raw_app/raw_app.yaml", "value: {}\n"); + await write(syncRoot, "f/a__raw_app/backend/config.script.lock", "x\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("keeps the reported path whole when the lint target is a path segment", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/conf/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, path.join(syncRoot, "f")); + + expect(report.issues.map((i) => i.path)).toEqual([ + "conf/orphan.script.yaml", + ]); + expect(report.issues[0].errors[0]).toContain("conf/orphan.script.yaml"); + }); + }); + + test("does not report a fileset child spelled like script metadata", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/data.resource.yaml", "value: {}\n"); + await write(syncRoot, "f/data.fileset/config.script.yaml", "a: 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); + + test("does not report a dbt project whose optional descriptor is absent", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/proj.script.yaml", METADATA); + await write(syncRoot, "f/proj__dbt/dbt_project.yml", "name: proj\n"); + await write(syncRoot, "f/proj__dbt/models/a.sql", "select 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); +}); diff --git a/cli/test/precheck_permissioned_as_apps_unit.test.ts b/cli/test/precheck_permissioned_as_apps_unit.test.ts new file mode 100644 index 0000000000..4a224a43c0 --- /dev/null +++ b/cli/test/precheck_permissioned_as_apps_unit.test.ts @@ -0,0 +1,147 @@ +/** + * The pre-check is what stops a push from silently reassigning an item's run-as + * user. Raw apps were missing from it, so the one kind whose file records no + * policy at all was also the one that changed owner without a word. + */ + +import { expect, test } from "bun:test"; +import { preCheckPermissionedAs } from "../src/core/permissioned_as.ts"; + +/** Non-interactive and without the override flag, the pre-check exits rather + * than reassigning silently — so a thrown exit is the signal it fired. */ +type Shape = "edited" | "added" | "deleted"; + +function change(path: string, name: Shape = "edited") { + return { name, path, before: "summary: x\n", content: "summary: x\n" }; +} + +async function precheck( + changes: ReturnType[], +): Promise { + const exit = process.exit; + let code: number | undefined; + (process as any).exit = (c?: number) => { + code = c; + throw new Error(`exit:${c}`); + }; + const logged: string[] = []; + const err = console.error; + console.error = (...a: unknown[]) => void logged.push(a.join(" ")); + try { + await preCheckPermissionedAs(changes, "pusher@corp", false, false, false); + } catch (e) { + if (!String(e).startsWith("Error: exit:")) throw e; + } finally { + (process as any).exit = exit; + console.error = err; + } + return code === undefined ? undefined : logged.join("\n"); +} + +test("a raw-app push warns the non-deployer it will take over the run-as user", async () => { + const message = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + + expect(message).toBeDefined(); + expect(message).toContain("f/test/myapp.raw_app"); + expect(message).toContain("pusher@corp"); +}); + +// Deleting one file re-pushes the whole app rather than deleting it, so the +// takeover happens there too. +test("deleting one of an app's files warns like editing one", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/gone.tsx", "deleted"), + ]); + + expect(message).toContain("f/test/myapp.raw_app"); +}); + +// The metadata file going with it means the app itself is created or removed — +// neither takes an owner over. +test("an app arriving or leaving whole is not a takeover", async () => { + const created = await precheck([ + change("f/test/new.raw_app/raw_app.yaml", "added"), + change("f/test/new.raw_app/index.tsx", "added"), + ]); + const removed = await precheck([ + change("f/test/old.raw_app/raw_app.yaml", "deleted"), + change("f/test/old.raw_app/index.tsx", "deleted"), + ]); + + expect(created).toBeUndefined(); + expect(removed).toBeUndefined(); +}); + +// An app carries no owner in its files, so nothing about it depends on their +// content — an empty one redeploys it exactly like any other. +test("an empty file still counts as a change to the app", async () => { + const added = await precheck([ + { name: "added", path: "f/test/myapp.raw_app/blank.ts", content: "" }, + ]); + const edited = await precheck([ + { name: "edited", path: "f/test/myapp.raw_app/blank.ts", before: "" }, + ]); + + expect(added).toContain("f/test/myapp.raw_app"); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `extractFolderPath` normalizes separators but the metadata predicates match a +// literal `/`, so a Windows path must not take a different branch. +test("a Windows path classifies the same as its posix twin", async () => { + const created = await precheck([ + change("f\\test\\new.raw_app\\raw_app.yaml", "added"), + change("f\\test\\new.raw_app\\index.tsx", "added"), + ]); + const edited = await precheck([ + change("f\\test\\myapp.raw_app\\index.tsx"), + ]); + + expect(created).toBeUndefined(); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `collectAppFiles` never sends these, and the sync diff never stops listing +// them (nothing uploads them, so they stay "added" forever) — so warning on one +// would gate every push of a scaffolded app on the override flag. +test("a file the push never sends is not a change to the app", async () => { + const artifacts = await precheck([ + change("f/test/myapp.raw_app/AGENTS.md", "added"), + change("f/test/myapp.raw_app/sql_to_apply/a.sql", "added"), + change("f/test/myapp.raw_app/node_modules/dep/index.js", "added"), + change("f/test/myapp.raw_app/recordings/r.json", "added"), + change("f/test/myapp.raw_app/package-lock.json"), + change("f/test/myapp.raw_app/wmill.d.ts"), + // Only the backend folder's *top level* is a runnable; nothing reads deeper, + // so the depth limit is what keeps a `backend/node_modules/` from becoming + // the perpetual diff this predicate exists to remove. + change("f/test/myapp.raw_app/backend/node_modules/dep/index.js", "added"), + ]); + // The three channels a push does send through: bundled file, metadata, runnable. + const sent = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + const meta = await precheck([change("f/test/myapp.raw_app/raw_app.yaml")]); + const runnable = await precheck([change("f/test/myapp.raw_app/backend/a.ts")]); + // The runnable channel is not the bundle: the bundle's name exclusions don't + // reach into it, so a runnable file sharing one of those names still deploys. + const namesake = await precheck([ + change("f/test/myapp.raw_app/backend/wmill.d.ts"), + ]); + + expect(artifacts).toBeUndefined(); + expect(sent).toContain("f/test/myapp.raw_app"); + expect(meta).toContain("f/test/myapp.raw_app"); + expect(runnable).toContain("f/test/myapp.raw_app"); + expect(namesake).toContain("f/test/myapp.raw_app"); +}); + +test("an app is listed once however many of its files changed", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/index.tsx"), + change("f/test/myapp.raw_app/raw_app.yaml"), + change("f/test/myapp.raw_app/backend/a.ts"), + change("f/test/low.app/app.yaml"), + change("f/test/low.app/inline.ts"), + ]); + + expect(message).toContain("2 item(s)"); +}); diff --git a/cli/test/push_diff_convergence_unit.test.ts b/cli/test/push_diff_convergence_unit.test.ts index 14e248c03e..67f6d5b4dd 100644 --- a/cli/test/push_diff_convergence_unit.test.ts +++ b/cli/test/push_diff_convergence_unit.test.ts @@ -64,6 +64,7 @@ async function diff( remoteEl: Mock, skips: Record, parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, + isEls1Remote = false, ) { const { changes } = await compareDynFSElement( localEl as any, @@ -76,7 +77,7 @@ async function diff( false, undefined, undefined, - false, + isEls1Remote, false, parentOwnsScheduleEnabled, ); @@ -296,3 +297,26 @@ test("push: checkout inline names stay inside the flow folder", async () => { await checkoutInlineNames(join(process.cwd(), "missing.yaml")), ).toEqual({}); }); + +// A push never applies the workspace's display name and applies its color only +// when the local file carries one (see pushWorkspaceSettings), so a file that +// differs only in what would not be applied is not a push change; a pull still +// rewrites the file. +test("push: settings.yaml differing only by name or an unset color is not a change", async () => { + const remote = local({ + "settings.yaml": "name: prod\ncolor: '#ff0000'\nerror_handler: null\n", + }); + const unsetColor = local({ + "settings.yaml": "name: staging\nerror_handler: null\n", + }); + const skips = { includeSettings: true }; + expect(await diff(unsetColor, remote, skips)).toEqual([]); + expect(await diff(remote, unsetColor, skips, undefined, true)).toEqual([ + "edited settings.yaml", + ]); + + const otherColor = local({ + "settings.yaml": "name: staging\ncolor: '#00ff00'\nerror_handler: null\n", + }); + expect(await diff(otherColor, remote, skips)).toEqual(["edited settings.yaml"]); +}); diff --git a/cli/test/push_workspace_settings_name_unit.test.ts b/cli/test/push_workspace_settings_identity_unit.test.ts similarity index 54% rename from cli/test/push_workspace_settings_name_unit.test.ts rename to cli/test/push_workspace_settings_identity_unit.test.ts index ef76ab7aec..c583a53282 100644 --- a/cli/test/push_workspace_settings_name_unit.test.ts +++ b/cli/test/push_workspace_settings_identity_unit.test.ts @@ -1,23 +1,32 @@ /** - * Regression guard: a pull (pushWorkspaceSettings) must not apply the workspace - * display name from settings.yaml. Rationale lives at the apply site in settings.ts. + * Regression guard: `sync push` (pushWorkspaceSettings) must never apply the + * workspace display name from settings.yaml, and must apply the color only when + * the file carries one. Rationale lives at the apply sites in settings.ts. */ import { expect, test, describe, beforeEach, mock } from "bun:test"; let changeWorkspaceNameCalls: unknown[] = []; +let changeWorkspaceColorCalls: unknown[] = []; let editWebhookCalls: unknown[] = []; let remoteName = ""; +let remoteColor: string | undefined = undefined; let remoteWebhook: string | undefined = undefined; // Every wmill.* call reachable from pushWorkspaceSettings is stubbed so the -// function runs without a backend; only the two we assert on record calls. +// function runs without a backend; only the three we assert on record calls. mock.module("../gen/services.gen.ts", () => ({ - getSettings: async (_a: { workspace: string }) => ({ webhook: remoteWebhook }), + getSettings: async (_a: { workspace: string }) => ({ + webhook: remoteWebhook, + color: remoteColor, + }), getWorkspaceName: async (_a: { workspace: string }) => remoteName, changeWorkspaceName: async (a: unknown) => { changeWorkspaceNameCalls.push(a); }, + changeWorkspaceColor: async (a: unknown) => { + changeWorkspaceColorCalls.push(a); + }, editWebhook: async (a: unknown) => { editWebhookCalls.push(a); }, @@ -30,7 +39,6 @@ mock.module("../gen/services.gen.ts", () => ({ editWorkspaceDefaultApp: async () => {}, editDefaultScripts: async () => {}, workspaceMuteCriticalAlertsUi: async () => {}, - changeWorkspaceColor: async () => {}, updateOperatorSettings: async () => {}, editDataTableConfig: async () => {}, editSlackCommand: async () => {}, @@ -40,13 +48,15 @@ mock.module("../gen/services.gen.ts", () => ({ const { pushWorkspaceSettings } = await import("../src/core/settings.ts"); -describe("pushWorkspaceSettings workspace name", () => { +describe("pushWorkspaceSettings workspace identity", () => { const ws = "phoenix"; beforeEach(() => { changeWorkspaceNameCalls = []; + changeWorkspaceColorCalls = []; editWebhookCalls = []; remoteName = "phoenix"; + remoteColor = undefined; remoteWebhook = undefined; }); @@ -69,4 +79,37 @@ describe("pushWorkspaceSettings workspace name", () => { expect(editWebhookCalls.length).toBe(0); expect(changeWorkspaceNameCalls.length).toBe(0); }); + + test("a settings.yaml without a color key does not clear the workspace color", async () => { + remoteColor = "#ff0000"; + remoteWebhook = "https://old"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + webhook: "https://new", + }); + expect(editWebhookCalls.length).toBe(1); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); + + test("a color in settings.yaml is applied when it differs from the workspace", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#00ff00", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls).toEqual([ + { workspace: ws, requestBody: { color: "#00ff00" } }, + ]); + }); + + test("a color matching the workspace is a complete no-op", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#ff0000", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); }); diff --git a/cli/test/raw_app_push_policy_unit.test.ts b/cli/test/raw_app_push_policy_unit.test.ts new file mode 100644 index 0000000000..fbbc7b55c0 --- /dev/null +++ b/cli/test/raw_app_push_policy_unit.test.ts @@ -0,0 +1,142 @@ +/** + * `raw_app.yaml` records none of the policy but the access-mode markers, so a + * push that regenerated the whole policy reset the deploy drawer's settings — + * run-as identity, sandbox isolation — to the pushing user's. Pin that the + * deployed policy is carried over, that a first push still starts from what the + * file states, and that the markers still close a deployed open app back down. + */ + +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { mkdtemp, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let calls: any[] = []; +let deployedPolicy: any; +/** No app deployed at the path: `getAppByPath` 404s and the push creates one. */ +let deployed = true; + +// Stub only what no other in-process suite imports, and treat a stub as +// permanent for the run (see "Module mocks" in cli/TESTING.md). These three API +// functions qualify — nothing else in `test/` imports them. `bundle.ts` did not: +// stubbing it left `raw_app_svelte_plugin_unit.test.ts` asserting against an +// empty bundle, which an `afterAll` hand-back did not prevent. So the real +// bundler runs instead, on the app each push writes below. +const realServices = await import("../gen/services.gen.ts"); + +mock.module("../gen/services.gen.ts", () => ({ + ...realServices, + getAppByPath: async () => { + if (!deployed) throw new Error("not found"); + return { + path: "f/test/raw", + summary: "raw", + value: { files: {}, runnables: {} }, + policy: deployedPolicy, + }; + }, + updateAppRaw: async (a: unknown) => { + calls.push(a); + }, + createAppRaw: async (a: unknown) => { + calls.push(a); + }, +})); + +// Belt and braces: nothing else in-process calls these, and a hand-back is not +// what makes that safe. +afterAll(() => { + mock.module("../gen/services.gen.ts", () => realServices); +}); + +const { pushRawApp } = await import("../src/commands/app/raw_apps.ts"); + +const ADMIN = { + userCache: new Map(), + userIsAdminOrDeployer: true, + userEmail: "deployer@windmill.dev", +}; + +async function push(yamlTail: string, admin = true): Promise { + calls = []; + const dir = await mkdtemp(join(tmpdir(), "windmill_raw_push_")); + await writeFile( + join(dir, "raw_app.yaml"), + `summary: raw\nrunnables: {}\n${yamlTail}`, + "utf-8", + ); + // Any file the remote doesn't have, so the push isn't short-circuited as + // up to date. It is also the bundler's entry point, so it has to compile. + await writeFile(join(dir, "index.tsx"), "export default 1\n", "utf-8"); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ name: "app", private: true }), + "utf-8", + ); + // `ensureNodeModules` only checks the directory is there; borrowing the CLI's + // own skips an npm install per push. + await symlink(join(process.cwd(), "node_modules"), join(dir, "node_modules")); + await pushRawApp("w", "f/test/raw", dir, undefined, "bun", admin ? ADMIN : undefined); + expect(calls).toHaveLength(1); + return calls[0].formData.app; +} + +beforeEach(() => { + deployed = true; + deployedPolicy = { + on_behalf_of: "u/svc", + on_behalf_of_email: "svc@corp", + sandbox: true, + frontend_sdk_scopes: ["jobs:run"], + execution_mode: "anonymous", + // Legacy v1 grants: the backend folds them into v2 at run time, so keeping + // them would keep granting runnables a push has removed. + triggerables: { "script/f/test/gone": {} }, + triggerables_v2: { "a:script/f/test/gone": {} }, + }; +}); + +test("a raw-app push keeps the deployed run-as and sandbox settings", async () => { + const body = await push("public: true\n"); + + expect(body.policy.on_behalf_of).toBe("u/svc"); + expect(body.policy.on_behalf_of_email).toBe("svc@corp"); + expect(body.preserve_on_behalf_of).toBe(true); + expect(body.policy.sandbox).toBe(true); + expect(body.policy.frontend_sdk_scopes).toEqual(["jobs:run"]); + expect(body.policy.execution_mode).toBe("anonymous"); + expect(body.policy.triggerables).toBeUndefined(); + expect(body.policy.triggerables_v2).toEqual({}); +}); + +test("a raw-app push without the marker closes an anonymous app back down", async () => { + const body = await push(""); + + expect(body.policy.execution_mode).toBe("publisher"); +}); + +test("a push that may not claim the deployed identity doesn't send it", async () => { + const body = await push("", false); + + expect(body.preserve_on_behalf_of).toBeUndefined(); + // Not just the flag: the identity itself stays off the wire, so no server can + // deploy this push under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + // Everything the pusher is entitled to carry over still comes along. + expect(body.policy.sandbox).toBe(true); +}); + +test("a first raw-app push deploys the policy its file states", async () => { + deployed = false; + const body = await push( + "policy:\n sandbox: true\n on_behalf_of: u/impostor\n on_behalf_of_email: impostor@corp\n", + ); + + expect(body.policy.sandbox).toBe(true); + // A repo doesn't get to pick who an app runs as: the identity never reaches + // the wire, so no server can be talked into deploying under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + expect(body.preserve_on_behalf_of).toBeUndefined(); +}); diff --git a/cli/test/schedule_push_permissioned_as_unit.test.ts b/cli/test/schedule_push_permissioned_as_unit.test.ts new file mode 100644 index 0000000000..17a1731494 --- /dev/null +++ b/cli/test/schedule_push_permissioned_as_unit.test.ts @@ -0,0 +1,143 @@ +/** + * Regression guard: the standalone `wmill schedule push` must resolve ownership + * the same way `wmill sync push` does. It only preserves the remote's + * `permissioned_as` when the command hands `pushSchedule` a context, so a push + * that builds none silently reassigns the schedule to whoever ran it. + */ + +import { expect, test, describe, afterAll, beforeEach, mock } from "bun:test"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let updateScheduleCalls: any[] = []; +let remotePermissionedAs: string | undefined = "u/svc"; + +const REMOTE_SCHEDULE = () => ({ + path: "u/admin/sched", + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: "u/admin/script", + is_flow: false, + args: {}, + enabled: false, + summary: "before", + permissioned_as: remotePermissionedAs, +}); + +// A module mock is process-global and outlives the file that installs it, and +// `mock.restore()` does not undo one: every mocked module has to be handed back +// its real exports here, or whichever file `bun test` happens to run next gets +// this file's stubs. +const realModules: [string, Record][] = []; +async function mockModule( + specifier: string, + factory: (real: Record) => Record +): Promise { + const real = { ...((await import(specifier)) as Record) }; + realModules.push([specifier, real]); + mock.module(specifier, () => factory(real)); +} + +afterAll(() => { + for (const [specifier, real] of realModules) { + mock.module(specifier, () => real); + } +}); + +await mockModule("../gen/services.gen.ts", () => ({ + getSchedule: async () => REMOTE_SCHEDULE(), + updateSchedule: async (a: unknown) => { + updateScheduleCalls.push(a); + }, + whoami: async () => ({ + email: "deployer@windmill.dev", + username: "deployer", + is_admin: true, + groups: [], + }), +})); + +await mockModule("../src/core/context.ts", (real) => ({ + ...real, + resolveWorkspace: async () => ({ + workspaceId: "w", + name: "w", + remote: "http://localhost/", + token: "t", + }), +})); + +await mockModule("../src/core/auth.ts", (real) => ({ + ...real, + requireLogin: async () => ({}), +})); + +const scheduleCommand = (await import("../src/commands/schedule/schedule.ts")) + .default; + +async function pushIn(wmillYamlTail: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_")); + await writeFile( + join(dir, "wmill.yaml"), + `defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`, + "utf-8" + ); + await writeFile( + join(dir, "sched.schedule.yaml"), + `schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`, + "utf-8" + ); + + const cwd = process.cwd(); + process.chdir(dir); + try { + await scheduleCommand.parse([ + "push", + "sched.schedule.yaml", + "u/admin/sched", + ]); + } finally { + process.chdir(cwd); + } +} + +describe("wmill schedule push ownership", () => { + beforeEach(() => { + updateScheduleCalls = []; + remotePermissionedAs = "u/svc"; + }); + + test("keeps the remote's permissioned_as under syncBehavior v1", async () => { + await pushIn("syncBehavior: v1\n"); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.summary).toBe("after"); + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + // The entry to read is the one matching the workspace being pushed to, not + // the top level: a repo that varies settings per workspace puts syncBehavior + // under `overrides` and nowhere else. + test("reads syncBehavior from the target workspace's overrides", async () => { + await pushIn( + `workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n` + ); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + test("leaves ownership to the backend below syncBehavior v1", async () => { + await pushIn(""); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBeUndefined(); + expect(body.preserve_permissioned_as).toBeUndefined(); + }); +}); diff --git a/cli/test/secret_bearing_deletions_unit.test.ts b/cli/test/secret_bearing_deletions_unit.test.ts new file mode 100644 index 0000000000..1fe2603527 --- /dev/null +++ b/cli/test/secret_bearing_deletions_unit.test.ts @@ -0,0 +1,66 @@ +/** + * The trashbin notice a push prints is only as good as its classification, which has + * to agree with the apply loop on two things: which deleted files are a variable or a + * resource — a path the loop skips must not be counted, or the notice announces a + * deletion that never happened — and that the unit is the server-side object, so a + * file resource's two files are the one deletion they cause. + */ + +import { describe, expect, test } from "bun:test"; +import { + secretBearingObjectKind, + describeSecretBearingChanges, +} from "../src/commands/sync/sync.ts"; + +describe("secretBearingObjectKind", () => { + test("matches variable and resource metadata in both serializations", () => { + expect(secretBearingObjectKind("f/test/protocol.variable.yaml")).toBe( + "variable", + ); + expect(secretBearingObjectKind("f/test/erp_access.resource.json")).toBe( + "resource", + ); + // A file resource's content file deletes the resource outright, so it counts. + expect(secretBearingObjectKind("f/test/conf.resource.file.ini")).toBe( + "resource", + ); + }); + + test("ignores files that only look like one", () => { + for (const p of [ + "f/test/my_type.resource-type.json", + // A fileset child can be any file; deleting one re-pushes the parent resource + // rather than deleting anything. + "f/test/data.fileset/edge/inner.resource.yaml", + "f/test/bar.script.yaml", + "f/test/foo.flow/flow.yaml", + // The apply loop skips a `.lock` deletion outright, so counting one would + // announce a deletion that never happens. Reachable for a resource type whose + // format_extension is literally `lock`. + "f/test/conf.resource.file.lock", + ]) { + expect(secretBearingObjectKind(p)).toBeUndefined(); + } + }); +}); + +describe("describeSecretBearingChanges", () => { + test("counts each kind separately and pluralizes", () => { + expect( + describeSecretBearingChanges([ + { path: "f/a.variable.yaml" }, + { path: "f/b.variable.yaml" }, + { path: "f/c.resource.yaml" }, + ]), + ).toBe("2 variables and 1 resource"); + }); + + test("counts a file resource's two files as the one resource they delete", () => { + expect( + describeSecretBearingChanges([ + { path: "f/c.resource.yaml" }, + { path: "f/c.resource.file.ini" }, + ]), + ).toBe("1 resource"); + }); +}); diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts index e9ea167ffc..38da094a37 100644 --- a/cli/test/sync_push_auto_metadata_repro.test.ts +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -168,7 +168,7 @@ test( // Customer scenario: a barrel file (f/lib/errors/index.ts) re-exports from // siblings (./types.ts, ./WorkflowError.ts, ...). An importer in a different // folder imports from the barrel. On a fresh DB, the dep job for the importer -// fetches index.ts via raw_unpinned + temp_script_hash, but bun's resolver +// fetches index.ts via raw + temp_script_hash, but bun's resolver // then has to resolve the barrel's *sibling* imports — and those need to be // in TEMP_SCRIPT_REFS too. test( @@ -268,3 +268,78 @@ test( }); }, ); + +// The importer and the leaf change; the barrel between them does not, so it is +// absent from the push's change set. See `resolvePlaceholdersFromLocal`. +test( + "sync push --auto-metadata succeeds when a changed leaf sits behind an unchanged barrel", + { timeout: 180000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml); + + await createLocalScript( + tempDir, + "f/barrel", + "helper", + "bun", + `export function add(a: number, b: number) { return a + b; }\n`, + ); + await createLocalScript( + tempDir, + "f/barrel", + "index", + "bun", + `export * from "./helper.ts";\n`, + ); + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { add } from "../barrel/index.ts"; +export async function main() { return add(1, 2); } +`, + ); + + const deploy = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (deploy.code !== 0) { + console.log("STDOUT:", deploy.stdout); + console.log("STDERR:", deploy.stderr); + } + expect(deploy.code).toBe(0); + + // Add an export to the leaf and use it from the importer. The barrel + // re-exports it already, so it stays byte-identical and out of the push. + await writeFile( + `${tempDir}/f/barrel/helper.ts`, + `export function add(a: number, b: number) { return a + b; } +export function subtract(a: number, b: number) { return a - b; } +`, + ); + await writeFile( + `${tempDir}/f/app/consumer.ts`, + `import { subtract } from "../barrel/index.ts"; +export async function main() { return subtract(3, 1); } +`, + ); + + const result = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("No matching export"); + expect(combined).not.toContain("Failed to generate lockfile"); + }); + }, +); diff --git a/cli/test/trash_commands.test.ts b/cli/test/trash_commands.test.ts new file mode 100644 index 0000000000..7b417dbc5e --- /dev/null +++ b/cli/test/trash_commands.test.ts @@ -0,0 +1,66 @@ +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { setupWorkspaceProfile, ensureFolder } from "./new_commands_helpers.ts"; + +describe("trash command", () => { + test("lists, shows and restores a deleted variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + await ensureFolder(backend, "test"); + const ws = backend.workspace; + const path = `f/test/trash_${Date.now()}`; + const api = (route: string, init: RequestInit = {}) => + backend.apiRequest!(`/api/w/${ws}/${route}`, { + headers: { "Content-Type": "application/json" }, + ...init, + }); + + let resp = await api("variables/create", { + method: "POST", + body: JSON.stringify({ path, value: "kept", is_secret: false, description: "" }), + }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + resp = await api(`variables/delete/${path}`, { method: "DELETE" }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + const list = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(list.code).toBe(0); + const item = JSON.parse(list.stdout).find((i: any) => i.item_path === path); + expect(item).toBeDefined(); + expect(item.item_kind).toBe("variable"); + + const get = await backend.runCLICommand( + ["trash", "get", "--json", String(item.id)], + tempDir + ); + expect(get.code).toBe(0); + expect(JSON.parse(get.stdout).item_data.row.value).toBe("kept"); + + // A bogus second id: the first restore must still go through, and the + // failure must show in the exit code. + const restore = await backend.runCLICommand( + ["trash", "restore", String(item.id), "999999999"], + tempDir + ); + expect(restore.code).toBe(1); + expect(restore.stdout).toContain(`variable '${path}' restored`); + expect(restore.stderr).toContain("999999999"); + + resp = await api(`variables/get/${path}`); + expect(resp.status).toBe(200); + expect((await resp.json()).value).toBe("kept"); + + const after = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(after.code).toBe(0); + expect(JSON.parse(after.stdout).some((i: any) => i.item_path === path)).toBe(false); + }); + }); +}); diff --git a/cli/test/variable_resource_push.test.ts b/cli/test/variable_resource_push.test.ts index e6bc84c326..b77ed48b50 100644 --- a/cli/test/variable_resource_push.test.ts +++ b/cli/test/variable_resource_push.test.ts @@ -361,6 +361,126 @@ describe("variable", () => { expect(content).toContain("is_secret: false"); }); }); + + test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/perms_var_${uniqueId}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: varPath, + value: "perms_test_value", + is_secret: false, + description: "Variable for extra_perms test", + }), + } + ); + expect(createResp.status).toBeLessThan(300); + await createResp.text(); + + const aclResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/acls/add/variable/${varPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ owner: "g/all", write: true }), + } + ); + expect(aclResp.status).toBeLessThan(300); + await aclResp.text(); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`, + "utf-8" + ); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir + ); + expect(pullResult.code).toEqual(0); + + const localPath = join(tempDir, `${varPath}.variable.yaml`); + const pulled = await readFile(localPath, "utf-8"); + expect(pulled).toContain("extra_perms:"); + expect(pulled).toContain("g/all: true"); + + const beforeResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const before = await beforeResp.json(); + + // Perm-only edit: downgrade the grant to read. + await writeFile( + localPath, + pulled.replace("g/all: true", "g/all: false"), + "utf-8" + ); + + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + const afterResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const after = await afterResp.json(); + expect(after.extra_perms).toEqual({ "g/all": false }); + // Routed through /acls/* rather than update_variable, so the row itself + // is untouched. + expect(after.edited_at).toEqual(before.edited_at); + expect(after.value).toEqual("perms_test_value"); + + // A yaml with no extra_perms field at all is "no opinion": a checkout + // that predates ACL sync must never revoke UI-managed grants. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`, + "utf-8" + ); + const noOpinionResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(noOpinionResult.code).toEqual(0); + + const noOpinionApiResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + expect((await noOpinionApiResp.json()).extra_perms).toEqual({ + "g/all": false, + }); + + // An owner present remotely but absent from a *present* map is revoked — + // the one direction that can destroy a grant. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`, + "utf-8" + ); + const revokeResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(revokeResult.code).toEqual(0); + + const finalResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const final = await finalResp.json(); + expect(final.extra_perms).toEqual({}); + }); + }); }); // ============================================================================= diff --git a/cli/test/workspace_key_filename_integration.test.ts b/cli/test/workspace_key_filename_integration.test.ts index ae22f191bf..a2e731eae5 100644 --- a/cli/test/workspace_key_filename_integration.test.ts +++ b/cli/test/workspace_key_filename_integration.test.ts @@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml"; import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts"; import { findResourceFile } from "../src/commands/script/script.ts"; -import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts"; -import type { SyncOptions } from "../src/core/conf.ts"; +import { + resolveWsNameForConfigFromFlags, + type SyncOptions, +} from "../src/core/conf.ts"; // Integration tests covering the bug where workspace-specific filenames used // the raw git branch name instead of the wmill.yaml workspace config key. diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index 047d50ee78..ac0b775e86 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -506,10 +506,23 @@ export async function deployItem( }, }); } else if (kind === "app" || kind === "raw_app") { - const app = await provider.getAppByPath({ + const rawApp = await provider.getAppByPath({ workspace: workspaceFrom, path, }); + // See the flow branch: a source-workspace principal is never valid here, and the + // policy carries the app's in `on_behalf_of`. Clearing it lets the backend derive + // the target's own from the address. A group travels as its synthetic + // `group-*@windmill.dev` address, which an admin-created account holding it would + // win: known and accepted, see `users::permissioned_as_from_email` in the backend. + const app = { + ...rawApp, + policy: { + ...rawApp.policy, + on_behalf_of: undefined, + on_behalf_of_email: onBehalfOf, + }, + }; if (alreadyExists) { if (app.raw_app) { const secret = await provider.getPublicSecretOfLatestVersionOfApp({ diff --git a/docker/RHEL8/Dockerfile b/docker/RHEL8/Dockerfile index c17c7685c2..ea53feabf1 100644 --- a/docker/RHEL8/Dockerfile +++ b/docker/RHEL8/Dockerfile @@ -33,6 +33,8 @@ COPY /backend/oauth_connect.json /backend/oauth_connect.json COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh COPY /system_prompts/auto-generated /system_prompts/auto-generated +# The flow chat imports the chat SDK's source (svelte.config.js alias `windmill-chat`). +COPY /chat-sdk/src /chat-sdk/src RUN cd /backend/windmill-api && . ./build_openapi.sh COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ diff --git a/docker/RHEL9/Dockerfile b/docker/RHEL9/Dockerfile index 3f24e936ab..ab94df5fc4 100644 --- a/docker/RHEL9/Dockerfile +++ b/docker/RHEL9/Dockerfile @@ -33,6 +33,8 @@ COPY /backend/oauth_connect.json /backend/oauth_connect.json COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh COPY /system_prompts/auto-generated /system_prompts/auto-generated +# The flow chat imports the chat SDK's source (svelte.config.js alias `windmill-chat`). +COPY /chat-sdk/src /chat-sdk/src RUN cd /backend/windmill-api && . ./build_openapi.sh COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md new file mode 100644 index 0000000000..ffec41e182 --- /dev/null +++ b/docs/ai-session-backups.md @@ -0,0 +1,400 @@ +# AI session backups + +AI sessions live in the browser: the session list (`windmill-sessions`), chat transcripts and +image blobs (`copilot-chat-history`) and artifacts (`copilot-artifacts`), all per-user IndexedDB +stores. This is the design of their backup in the workspace's object storage, and the +constraints future work on either side must keep. + +Backend: `backend/windmill-api/src/ai_sessions.rs` (`/w/{w}/ai/sessions/{list,pull,push}`). +Frontend: `frontend/src/lib/components/sessions/sessionMirror*.ts`. + +## Why it is lazy + +A session changes at the local write rate: a transcript write every 2 s while streaming, a +session-record write per new message on screen. An object in S3 is replaced whole and every PUT +is billed, so the backup deliberately does not follow that rate. Local writes only mark a session +dirty (`sessionMirrorSignal.ts`, import-free so the stores never depend on the backup). A flush +runs 15 s after the marks go quiet, at most 2 min after the first unflushed mark, when the tab is +hidden, and 10 s after load for marks a crash left behind. Marks are persisted in localStorage +(shared by the user's tabs) for that reason, one key per mark: a shared blob would let two tabs +marking different sessions at once rewrite each other's mark away. A dirty mark is a counter +bumped on every write; a push retires it by recording the counter it covered on the session's +sync row rather than deleting the mark, since two localStorage calls cannot compare-and-delete +and a bump landing between them would be lost; retired marks are not reclaimed (one small key per +session ever backed up), and the marks of unsent drafts and of workspaces that are off stay too, +each costing one lookup per flush. Only a session gone from the store has its mark deleted. Losing the last +seconds of a device that never comes back is accepted; a tab that closes normally keeps its marks. +A signal names the user whose store the write landed in (read off the store's scoped name), so +a write that completes after the logged-in user changed marks that user's session, for their +next load, rather than the current user's. + +A flush plans and sends one session at a time, filling requests of about 8 MB as it goes, so a +first backfill of a large history never holds more than one request's worth of records and +images in memory. + +## What a push carries + +The pure planner (`sessionMirrorPlan.ts`) compares each piece against the marker of what was +last pushed, kept per session in the `windmill-sessions-mirror` store: + +| Piece | Object | Sent when | +|---|---|---| +| session record | `sessions/{sid}/head.json` | its signature changed | +| chat | `sessions/{sid}/chats/{cid}.json` | its `lastModified` moved | +| artifacts | `sessions/{sid}/artifacts.json` | their fingerprint changed | +| image | `images/{sid}/{cid}/{iid}` | never pushed before (write-once) | +| index marker | `index/{sid}/{epoch}` | last, by the part that completes a push of the session (empty; named by the record's move count) | + +All under `windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/` in the workspace's +primary storage (the generation is what a key rotation moves, see below). +The listing reads only `index/`: one object per session whatever the session holds, so a +session with many chats cannot crowd newer ones out of a bounded scan, and its +`last_modified` is the session's `updated_at`. Written last, and only by an entry no unsent +part follows (a session split over several entries says `partial` on all but the last, and +names the push on each, or the part is refused), it +lists a session only once a whole push landed; the parts of a session after a failed one are +not written either, on the server within one push and on the client across pushes, so the +marker on the last part never lists a session missing a chat, and a new session whose last part +never lands is not listed at all. A push of the session whole (no sync row, or a stale one, +`whole` on every part) opens with the head on the first: that part replaces the backup (the +marker goes first, then everything under the session), so what an old storage still held of +the session and the push does not carry is gone. An incremental part rides on a listed +session, and the server refuses it with `needs_whole`, writing nothing, when none is listed +(a removal deletes the marker first), rather than write a marker over a session missing what +earlier parts or earlier pushes carried; one that changes more than one object unlists the +session before its writes and lists it again after them, so a pull between two of the writes, +or after one failed, finds it absent rather than a mix of old and new pieces (one object +changing is one write, and stays listed). A push split over several parts, whole or +incremental, names itself on each with a token the browser draws (`push`, `opens` on the +first): the opening part unlists the session, so a pull between two parts finds it absent +rather than a mix of old and new pieces, the last part lists it again, and a later part is +written only while that token is the one there, so two devices pushing the session at once +cannot list a mix of their pieces (the push that opened later wins; the other is refused with +`needs_whole` and goes again), and one abandoned leaves the session unlisted, so the next +push of it goes whole. A push and a removal of one session +are serialized on the server by a Postgres advisory lock keyed on the session's prefix, so +the two never interleave object by object. + +The head signature leaves out `name` (a per-browser counter the sessions page routes by), +the unsent-draft fields, `workspace_root_id` (recomputed on import), and the two fields reading +a session bumps (`lastSeenCount`, `lastActivityAt`). Reading a session must never cost a push; +keep that property when adding fields to `Session`. + +Unsent drafts (no `workspace_id`) and attached files (Blobs, directory handles) are not backed up. + +## Encryption and access + +Every object is encrypted with a key derived from the workspace key and the user +(`build_crypt_with_key_suffix` with the email hash), because workspace storage credentials are +shared far more widely than a user's transcripts: `public_resource` storages and legacy-mode +READ/WRITE hand any member the bucket. The key is per user rather than per workspace so that a +member who copies another user's ciphertext under their own prefix gets nothing from `pull`; an +object that does not decrypt for its reader is treated as absent. Rotating the workspace key +(`set_encryption_key`) does not re-key the backups the way it re-encrypts the workspace's +secrets. The objects live under a prefix named by a generation +(`workspace_settings.ai_sessions_backup_generation`) that the rotation bumps in the +transaction committing the new key; once committed, the routes read and write under the new +generation's prefix, the answers name it (`backup_generation`, below; `storage_id` names the +storage and does not change), so every browser marks its sync rows stale and pushes its +sessions whole again there, and every older generation, which nothing writes to any more, is deleted off the +request at leisure (`windmill-api-workspaces/src/ai_session_backups.rs`). Sessions no browser +holds any more are lost. A generation is never reused, so no deletion, however late, can +touch live objects; a rotation that fails before its commit bumps nothing and deletes +nothing; two rotations racing serialize on the key row; the same key set again bumps +nothing. A rotation is rare, and the alternative, +rewriting every object in place while pushes, restarts, storage switches and further +rotations race the rewrite, is where the complexity would be; with this, nothing but the +current key ever reads an object. The +server builds every key from ids it validated +(`[A-Za-z0-9_-]{1,64}`) and the caller's own email; the client never names a key, and the +workspace storage permission rules are not consulted (the same stance as volumes). Only an +unscoped user token may reach the routes: a job token can carry an `on_behalf_of` identity and +every scoped token (guest, embed, app policy, MCP) was minted for something narrower. + +The backup is keyed by the email like the browser's own stores are (`userScopedDb` scopes +IndexedDB by it): a user whose email changes starts from an empty history on both sides, and +the objects under the old hash stay in the bucket unread. Carrying them over would need a +server-side re-key (decrypt with the old suffix, encrypt with the new, move every object) in the +email-change flow, which this design leaves out. + +An image is accepted only as a base64 data URL of at most 4 MB and stored verbatim, so it +serializes back into a pull answer at its stored size; anything JSON would escape could grow +several times and defeat the pull budget. + +`push` carries `owner`, the email the browser prepared the batch for, and the server refuses a +mismatch with 409: an in-place account switch must not file one user's sessions under another's +prefix. The client captures its user at flush start and checks every store handle's name +against it for the same reason. + +The feature is on wherever the workspace has primary storage, and off with +`ai_config.sessions_storage_disabled` (the `copilot_disabled` pattern: no migration, carried by +settings export and the CLI). A build without `parquet` has no routes (404), a workspace without +storage and nothing to stand in for it answers `enabled: false`; either turns the backup off +for ten minutes, after which the +page asks again on its own (a flush for whatever is pending, and a restore), and the AI +settings page tells the mirror at once when the switch is saved there (the off state is +forgotten, the rows that went stale are marked again, a restore runs). + +## The instance store standing in + +A workspace without storage of its own keeps its backups in the instance object store +(`object_store_cache_config`, loaded the way every other use of it is, so never with +`DISABLE_S3_STORE`; the plan is checked on every request and Pro never falls back, since a +store loaded before a switch to Pro stays loaded), under the same layout and the same +per-user key, +while the instance setting `ai_sessions_instance_storage_fallback` allows it (on unless set +to false; the instance settings page shows it under Object Storage). A build without +`private` has neither workspace storage nor the quota below, and never falls back. Every +answer says which kind of store it came from (`fallback`), and the instance store is named +(`storage_id`) by what locates its objects, the endpoint, region and bucket its settings +resolve to, in a namespace of its own: moving the instance store to another endpoint under +the same bucket name is a storage switch for the browsers, and a workspace bucket is never +taken for it. The location is kept with the loaded store, so a server whose reload is still +pending names the store it writes to. A route decides between the workspace's storage and +the instance store from the row it reads the generation from, so a push lands in the +instance store only under a generation read while the workspace had no storage. The store a +workspace's backups live in is resolved in one place +(`ai_session_backups::workspace_store`), for the routes and for the rotation's deletion of +older generations alike. + +Configuring a storage for a workspace that had none (`edit_large_file_storage_config`) bumps +the backup generation in the transaction that sets it, so everything the workspace left in +any instance store sits under a generation the routes never read again: a later return to +the instance store, whichever it is by then, starts from a newer one. Every storage settings +change then deletes from the instance store, off the request, the workspace's generations +older than the one it committed, whether the setting is on or off (copies from when it was +on may be there). Nothing live is older, whatever happens next: a deletion that is slow, cut +short, or overtaken by the storage being dropped or pointed at the instance store's own +bucket touches only generations nothing reads. Dropping the storage bumps nothing and is a +switch like any other: the rows go stale, the sessions are pushed whole into the instance +store, and the old bucket keeps its copy. On the browser side a removal owed to an instance +store (the row names it apart, `storageName` in `sessionMirrorPlan.ts`) is retired by any +answer from the workspace's own storage, since that storage being there means the +generation moved past the copy; one owed to a workspace storage still waits for that +storage, whatever the instance store answered. Copies a deletion missed stay in the +operator's bucket unread, as a deleted workspace's copies do. + +On CE the bytes in the instance store count toward the workspace's storage quota under a +storage name of their own (`_ai_sessions_fallback_`, listed by the periodic recount while +the workspace has no storage of its own, and left out when there are none), so a member +cannot fill the operator's bucket past what the workspace may use; on EE, where workspace +storage has no quota either, nothing bounds them but the per-push caps and the instance +setting. + +## Conflicts and deletion + +Last write wins across devices. The head carries no manifest; `pull` lists the session's prefix +instead, so a stale device that renames or archives a session rewrites only the head and cannot +hide chats a newer device wrote. Two devices continuing the same chat still collide. + +Restore brings back only sessions the browser does not have (`importSessions` is write-if-absent, +and skips ids the user deleted in this page) and never overwrites or deletes a local one from +remote state. It covers the workspace and its forks together, and only once every one of them +that keeps backups has listed (a listing that failed leaves the family for the next page load +or workspace switch, or the copy that did list could be the stale one): a session listed by +two of them (moved between them, the old copy not yet removed, since that mark is the moving +browser's, which may never come back) is brought back from the copy that moved last (`epoch`, +the record's move count, which names the marker), the storage's own modification time +deciding between two of the same count, and not from the other, which would otherwise take +the id first and keep the later copy out for good. A workspace's records land together once +its pulls are done, and just before they do the whole family is listed again (members whose +backups were off included, since a move from another device can land in a workspace between +the first listings and the pulls; a family of one, with nowhere else for a copy to show up, is +not): a session a later copy of which showed up elsewhere is left, with the family, for the +next time. Only a user-initiated `deleteSession`, and the retention sweep below, remove the +backup; the next push from +another device that still has the session is refused with `needs_whole` (nothing of it is +written), its row goes stale without a backoff, and that device's next flush sends the session +whole; the workspace-lifecycle +removals (`reconcileSessionsLifecycle`, `deleteSessionsForWorkspace`) leave it, so a session +dropped by a wrong reconcile comes back on the next restore. Objects of deleted workspaces stay +in the bucket. A session moved to another workspace is pushed whole into the new one, and once +that push has landed the copy in the old one gets a removal mark of its own, naming the storages holding +that copy (the row that knew is the new workspace's by then), retried independently until +each of them has answered, even when the old workspace's backups are off at the time (they +may hold the copy still). Filing the removal only after the new copy is acknowledged keeps the +session backed up somewhere at every point. + +A restore writes a session's artifacts and chats before its record, and records nothing for a +session whose pieces could not be written: recording it would let the next flush push the +half-empty local state over the backup. + +Every answer names the storage it came from (`storage_id`, a hash of what locates the objects, +endpoint, region and bucket, not the credentials, which rotate; the instance store standing +in for a workspace without one is named apart, see above) and the backup generation a +key rotation bumps (`backup_generation`). A sync row records both, and a row naming another +storage or generation goes stale and its session is marked again: a workspace pointed at a +new bucket, or whose key was rotated, holds nothing, and the server looks nowhere else, so +the next flush carries the session whole. A switch leaves the old copy where it was, so the +row rewritten under the new storage records the old one (`alsoIn`, one entry per storage +the workspace was on), and a removal is done only once every storage holding a copy answered +it, whatever the generation (a rotation deleted the older generation's copy anyway): each +answer narrows the row to the storages still holding one, and the mark waits for them to +answer, so a switch back never brings a deleted session back. That includes the rows a flush has just written, when a later answer of the +same flush names another storage or the session was pushed in part on top of a row from the +old one; a session whose own parts were answered from different storages is not settled at +all. The listing a restore starts with runs the same check, so a storage switch is noticed at +the first push after it or on the next page load, whichever comes first. + +## Retention + +`ai_config.sessions_retention_days` (per workspace, in the AI settings; unset by default; +the `sessions_storage_disabled` pattern: no migration, carried by settings export and the +CLI; 1 to 3650) puts an age on sessions, counted from their last activity. Each side applies +it with its own clock against its own timestamps, so no clock is compared with another +machine's, and the two do not time the same event: the server counts the last push that +completed, a browser its last local activity, which includes reading new messages and is not +pushed. A backup swept while a browser still reads its copy comes back once that browser +writes to the session again (its incremental push is refused and goes whole): + +- The server sweeps the object store (`sweep_expired_ai_session_backups`, from the monitor + about every 40 minutes on each server, one pass at a time under a session-level advisory + lock). For every workspace with a retention it takes the store its backups live in, its + own storage or the instance store standing in, decided from the row it reads the + generation from as the routes do, names the users under the generation prefix + (`list_with_delimiter`) and lists each user's `index/` once: one object + per session, nothing of what the sessions hold. A session whose marker is older than the + retention is removed under its lock (`lock_session`), once its markers, listed again + there, are still all older: a push that renewed the session between the walk and the lock + keeps it, and one split over parts either holds the lock or has the session unlisted with + its token next to the markers (`index/{sid}/push`), which the sweep leaves alone while the + token is younger than the retention: an older one is a push a browser abandoned, whose + landed parts nothing lists, and it goes the same way. Before deleting anything the sweep + writes a record next to + the markers (`index/{sid}/sweep`, not an epoch, so neither `list` nor `pull` counts it), + and `remove_session` deletes it last: a removal cut short, its markers already gone, is + found by the next pass and finished, unless a push listed the session again first. At + most 1000 sessions per workspace and pass; the rest wait for the next. `list` leaves an + expired marker out of its answer meanwhile, so a browser never restores a session the + sweep has not reached. The marker's modification time is the storage's clock and the + cutoff the server's. The sweep reaches only the backups the routes would: a deleted + workspace's stay in its storage, and so do those a workspace keeps in the instance store + once `ai_sessions_instance_storage_fallback` is set to false. +- The browser sweeps its own stores when a tab resolves the logged-in user + (`sweepExpiredSessions`, from the one `onUserChange` in `sessionState.svelte.ts`), before + that tab reads a single session. A session whose last activity is older than the retention + by the browser's clock is deleted locally, record, chats, images, attached files and + artifacts. A restored session carries the backup's time as its last activity, the storage's + clock, so it counts from the later of that and the moment it was restored here + (`restoredAt`): a browser clock ahead of the storage's never deletes a session it just + brought back. Archived sessions count like any other, and persisted unsent drafts by their + pending workspace. + + The stores are shared by the user's tabs, and each keeps copies of the sessions in memory, + so every tab holds a shared Web Lock from before it reads them until it stops using them, + and the sweep deletes only while holding that lock exclusively, requested if available: + granted exactly when no tab of the user has the sessions loaded, which is why the sweep + runs where it does and nowhere else. Nothing holds a copy of what it deletes and nothing + writes the stores meanwhile, so it deletes one record at a time and without re-reading. It + also takes the tab lock the flush and the restore take, again only if available, so neither + plans nor stages a session half deleted; like the restore, it does not run where Web Locks + do not exist. With several tabs open nothing is swept, until one of them reloads alone. + + The hold is only as good as the tabs that take it, so a tab still running a build from before + it has the sessions loaded and holds nothing. A tab loaded after that one, across a deploy, + can sweep a session the older tab has in memory, and a write there afterwards brings the + record back without its chats, which the next flush pushes. It needs a tab left open across a + deploy, a session untouched for the whole retention, and the user going back to that session + in the older tab; the next sweep deletes it again. The same window is open to the + workspace-lifecycle delete in `reconcileSessionsLifecycle`, which no lock guards at all. + + What deletes is the retention the server gives as the sweep runs, asked for under both locks + (`POST /workspaces/session_workspace_retention`, its own route rather than a field on the + lifecycle status, whose answer a tab loaded before this version still reads). Never a + remembered one: a retention raised or cleared since would otherwise delete a session that is + now within it, and a persisted unsent draft has no backup to come back from. What the sweep + keeps in localStorage decides only whether to ask again — it asks when it has asked nothing + yet, when the answer it has is a day old, or when that answer marks a session expired — so + an ordinary load costs no request at all. An answer that does not arrive within five seconds + leaves the sessions for the next load rather than delete on what this browser guessed. That + route answers for a workspace the caller can be authed into, unlike the status: a status is + what to do with the caller's own sessions, a setting is the workspace's to tell, so a + disabled membership is told nothing though its sessions still reconcile. + + Each session's record goes before its pieces, so nothing plans a push for it afterwards, + and a localStorage key written before the record and removed once every piece is gone makes + a later sweep finish a deletion that failed, unless a restore brought the session back + since. The record is deleted without the tombstone a user delete leaves, which is what lets + a restore bring it back. The session's dirty mark and sync row go with it (`sessionSwept`), + unless the row still carries a removal or a restore's staging. Nothing is sent to the storage: the local + copy's age says nothing about another device's, which may have pushed the session since, + and the server applies the rule to the backup on its own. A session swept here that the + storage still lists comes back on the next restore. + +## Limits + +Push bodies are packed to about 8 MB (UTF-8 bytes as sent), at most 100 entries, 200 removals and +4000 pieces each (the server's caps, with 32 MB on the body, and 100 chats, 500 images or 1000 +deletes per entry, since every piece is an object-store call); an entry that +outgrows the target is split into chat-only parts (the artifacts and deletes on the last, the +head on the last too for an incremental push and on the first part, whatever it carries, for a +push of the session whole), and deletes +past the per-entry cap are carried over to the next push, which the session stays marked for. A chat above +16 MB or a session's artifacts above 8 MB are left out with a console warning; a chat that grew +past the cap after it was backed up has its copy deleted, so a restore never presents the old +transcript as the current one. A 413 fails only the sessions of that request. A +request the server refuses (any other 4xx but 404/403/409) stops the backup for the page but keeps +the marks and the sync state, so the next load tries again; a session the server reports it could +not store stays marked and is retried with backoff. A move files the old workspace's removal +mark before recording the new copy's row, so a mark that could not be written leaves the move +to be planned again. A workspace that answers `enabled: false` +marks its sync rows stale (the next push after storage returns carries every session whole, +since a new storage may be a new bucket) and leaves its dirty marks where they are (a move into +it must still remember the old copy); it keeps the removal marks of sessions that had been backed +up, so one deleted while backups are off does not come back once they are on, and drops the +removals of sessions never backed up from this browser, so a storage-less instance does not +collect one mark per deleted session forever. A user delete whose removal mark cannot be written +(localStorage full) is carried by the session's sync row instead (`removed`), which the flush +and the restore read like a mark; a session without a row yet (its first push may be in +flight) gets a row saying only that, and every row write keeps a removal filed meanwhile, so +the push's own row cannot erase it. Pull bodies are +capped at 64 KB. Pull answers up to 20 ids within a 32 MB +budget: a session's size is known from the listings before anything of it is read, one that +would not fit is deferred unless it is the first of the answer, in which case it comes in +pages: the answer carries what fits in key order (at least one object, so every page makes +progress) and names where the next picks up (`next`, a cursor the browser sends back as +`resume` with that session alone). Every page carries a fingerprint of the session's listing +(marker, keys, sizes, modification times, entity tags and versions, since a store reports +modification times coarsely and an object rewritten at the same size within that grain would +otherwise fingerprint the same) taken before anything of it is read, and the server +takes it again once the page is read: a page the backup moved under (a push landing object by +object) is read again, a few times, then answered as `moved`, and the browser starts the +session over on that or on two pages whose fingerprints differ. The browser writes each page's pieces as it arrives, over whatever +an earlier restore cut short had staged (the session is absent locally, so its pieces have no +local edits to keep, and the backup may have moved on), and the record, which is what makes +the session visible, only with the last page. A restore in progress keeps a staging row for +the session (the ids of every chat, image, artifact and version it wrote), which outlives it +if it is cut short; the next restore deletes the staged pieces the backup no longer has, by id +and never by clock, before the record lands (once the record is there no restore looks at the +session again, and a flush would push them back), and a prune that could not run leaves the +session, its pieces and its staging row for the restore after. A restore holds the user's tab +lock while it runs, so two tabs cannot each write the same absent session's pieces over the +other's, and runs only where Web Locks exist (a secure context: https, or localhost); on a plain +http origin the browser still backs up, and its sessions come back on a secure one. A +restore never writes an older record over a newer +one; between pages it holds nothing but the sync +row being assembled, whose chats also admit the images of a later page. Every page carries a +fingerprint of the session's listing (`listing`), taken before anything of the page is +listed or read, so an object landing after it is in the next page's; a session whose +fingerprint moved between two of its pages (a chat added by another device could sort before +the cursor and be missed) starts over, up to three times, then waits for the next restore. An object that grew +since the listing (a push replaced it) ends its page just before it and the answer names that +spot, so the next page sizes it anew rather than the session being imported without it. A pull sees every key of a session's listing but keeps the 5000 smallest +past its cursor (a page is defined by key order, and the store promises none), so a session +grown without bound by valid pushes cannot grow the answer's memory through its metadata +either; removing a prefix and a rotation's deletion stream their listings. `list` scans at most 50 000 index markers, keeps the newest 500 as it goes and answers with +them (`truncated` says when there were more); the restore takes 50 of them. Every read checks the object's size before buffering it: one larger than any push writes +(32 MB) is planted, whatever its listing said, and skipped, since whoever holds the bucket's +credentials can put anything at a predictable key; one larger than its listing said grew +since (a push replaced it) and ends its page, for the next pull to size anew. A dirty mark that cannot be written +(localStorage full) records its bump on the session's sync row instead (`extraV`, counted +with the mark's counter and kept by every row write, so a push in flight cannot retire it); +a session without a row yet (its first push in flight) keeps the bump in the page, and the +next row write takes it onto the row in the same transaction, so the row the push writes +cannot retire the mark with the bump unseen; every load's backfill marks again a session +without a row, with a stale one, or with one carrying bumps no push has covered. +A mark or removal for another user (a write that landed after a switch) reaches that user's +rows through a connection of its own, since the shared handle follows the current user. Nothing is read past the budget, whatever a session holds. A +restore takes the newest 50 sessions per workspace: every visible session gets a runtime, and +each runtime's history load reads the whole chat store. On CE the push checks the storage quota +and bumps usage by bytes written (an over-count on overwrites; the periodic recount settles it). diff --git a/docs/app-policy-email-removal.md b/docs/app-policy-email-removal.md new file mode 100644 index 0000000000..fb4cf100fd --- /dev/null +++ b/docs/app-policy-email-removal.md @@ -0,0 +1,47 @@ +# Removing `policy.on_behalf_of_email` + +An app's identity is `policy.on_behalf_of`; the address beside it is a function of that +principal. `on_behalf_of_email` is no longer required — a policy carrying only a principal +executes, deriving the address — but it is still written on every save, and that is the only +thing holding it in place. + +## The gate + +`get_on_behalf_of` gained its derive-when-absent fallback in **1.810**. Every replica before that +*requires* the key and errors outright without it, so it would 400 every anonymous, publisher and +guest app saved by a newer one. A rolling deploy runs both versions at once, which is why the +write stays until no replica older than 1.810 can be live — in practice, once +`MIN_KEEP_ALIVE_VERSION` (`windmill-common/src/min_version.rs`) has passed it. + +There is no `MIN_VERSION_*` constant for this and it does not need one: those exist to gate +behavior at runtime or to trip the build when a constraint expires, and nothing here does either. +The key is written unconditionally, so no replica ever meets its absence until someone follows +the steps below. + +## Step 1 — stop writing the key + +- `stored_on_behalf_of_email` in `windmill-api/src/apps.rs`, and the `create_app` / + `update_app_internal` call sites that store what it returns. +- The CLI and frontend workspace-deploy paths (`cli/windmill-utils-internal/src/deploy.ts`, + `frontend/src/lib/utils_workspace_deploy.ts`). These send the address *instead of* a principal + for a cross-workspace deploy, which is the one case where it is the only identity available — + so this is "stop sending it once the target resolves a principal itself", not a deletion. + +Policies written before this keep their key and keep being read from it; they agree with their +principal, so nothing has to strip them. + +## Step 2 — drop the field + +Remove `on_behalf_of_email` from `Policy` and the fallback in `get_on_behalf_of`, which then +always derives. Optionally strip the key from stored policies. + +This can ship with step 1. It is written separately because step 1 alone is revertible without +touching stored data or the response schema, and because the gate above is what makes either +step safe — nothing about step 2 needs its own waiting period. + +## Why the address is not derived on read + +Read paths return the stored address verbatim rather than recomputing it. Deriving on read means +resolving a principal that, for a draft, is caller-controlled — which turns the read into an +oracle for addresses the caller cannot otherwise see, and leaves a principal that resolves to +nobody with no address at all. Both were live defects while the read paths did derive. diff --git a/docs/auth-surface.md b/docs/auth-surface.md new file mode 100644 index 0000000000..0876a1184a --- /dev/null +++ b/docs/auth-surface.md @@ -0,0 +1,50 @@ +# Auth surface: facts that are easy to get wrong + +Symbols, not line numbers, are cited: they drift less. + +- **Credential precedence** (`windmill-api-auth/src/auth.rs` `extract_token`): `Authorization: Bearer` + → `token` cookie → `?token=` query param. A URL with `?token=` is a credential on every route, but + an existing cookie silently wins over it. +- **`AUTH_CACHE`** caches a token's identity for 120 s. Deleting a token row does not purge it: the + DB trigger (`migrations/20260316000001_token_hash_pk_swap.up.sql`) notifies only for + `label = 'session'` rows, and `delete_token` never calls `invalidate_token_from_cache`. +- **Sessions** are `token` rows with `label='session'` plus the HttpOnly `token` cookie, minted only + by `create_session_token` (`windmill-api-users/src/users.rs`). `GET /api/users/refresh_token` + mints one for any non-job token but returns plain text, no redirect. +- **`tokens/impersonate`** (superadmin) returns a multi-use token and sets no cookie. +- **Every superadmin route refuses a job token**: `require_super_admin` + (`windmill-api-auth/src/lib.rs`) errors on `authed.job_id.is_some()`. A script that needs + `users/create`, `tokens/impersonate`, `set_login_type`, … must use a dedicated superadmin user + token stored as a secret, never `$WM_TOKEN`. Token scopes cannot narrow superadmin routes. +- **`login_type`** (`password` table) is a free-form `VARCHAR(50)`. Password login and password + reset require `login_type = 'password'`; `set_password` also accepts `pending_oauth` and turns + the account into a `password` one in the same statement (an account created ahead of its owner + gets its first credential that way, or through the OAuth claim below). +- **Login links** (`login_link` table, `POST /users/login_links` superadmin-only, + `GET /auth/login_link/{token}` unauthenticated): single-use, ≤15 min, a session cookie and a + 302 to a same-origin `rd`. `require_login_type` on the mint refuses (409) an account whose + `login_type` has moved on — the way a caller re-entering an account it created stops being + able to once the owner has a password or a provider. +- **Pre-approved trial offer** (`cloud_trial_offer`, cloud-only routes under + `/users/cloud_trial_offer`): written by a superadmin at provisioning, consumed by + `{consumed: true}` or by the portal's refusal; `…/go` is the one Windmill→portal hop that + mints a portal login, over the same `CUSTOMER_SERVICE_TOKEN` trust the onboarding hook uses + (`users_ee.rs`, the portal's admin token). It never expires on its own. +- **OAuth login** (`oauth2_ee.rs` `login_externally`, decision in `existing_login_decision`) + matches an existing account by lowercased email only. Same provider → login; a + `pending_oauth` account (see `PENDING_OAUTH_LOGIN_TYPE`) is **claimed** by the first login + whose address the provider itself asserted and did not mark unverified — `login_type` becomes + the client key and the hash is nulled; otherwise `require_preexisting_user_for_oauth` decides: + on, *every* existing account is loggable-into by any provider; off, "exists but with a + different login type". A new account gets `login_type = `. +- **OAuth email trust**: `LoginUserInfo.email_verified` is read leniently (bool or + "true"/"false" strings) and is only consulted for the claim above; only GitHub is filtered to + `primary && verified`; a missing email is fabricated from `name` as `@windmill.dev` and + reaches `login_externally` with `email_asserted = false`. +- **`GET /api/oauth/login/{client}`** is an unauthenticated 302 to the provider — a plain link + from any page starts SSO. +- **`CLOUD_HOSTED`** is presence-tested (`windmill-common/src/worker.rs`): `CLOUD_HOSTED=false` + still enables cloud mode. Of the routes above only the cloud trial offer and onboarding + profile routes are cloud-gated; for the rest, cloud only adds quotas. +- **`CREATE_WORKSPACE_REQUIRE_SUPERADMIN`** defaults to `true` when unset; only the literal + `"true"` enables it when set. diff --git a/docs/clone_repo_and_upload_to_instance_storage.bun.ts b/docs/clone_repo_and_upload_to_instance_storage.bun.ts deleted file mode 100644 index 485ac0c766..0000000000 --- a/docs/clone_repo_and_upload_to_instance_storage.bun.ts +++ /dev/null @@ -1,390 +0,0 @@ -import * as wmillclient from "windmill-client"; -import { basename, join } from "node:path"; -import { existsSync, rmSync } from "fs"; -import process from "process"; -import { spawn } from 'child_process'; -import * as fs_async from 'fs/promises'; -import * as fs from 'node:fs'; - -const UPLOAD_CONCURRENCY = 16; -const CLONE_MARKER_FILE = ".windmill_clone_complete"; - -type GitRepository = { - url: string; - branch: string; - folder: string; - gpg_key: any; - is_github_app: boolean; -}; - -export async function main( - resource_path: string, - workspace: string, - git_ssh_identity?: string[], - commit?: string -) { - let clonedRepoPath: string | undefined; - - try { - console.log("Starting git clone and Blob storage upload process"); - - // Get the git repository resource - const repo_resource: GitRepository = await wmillclient.getResource(resource_path); - - const cwd = process.cwd(); - - if (git_ssh_identity) { - process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity) - } - - // Handle GitHub App authentication if needed - if (repo_resource.is_github_app) { - const token = await get_gh_app_token(); - repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token); - } - - process.env["HOME"] = "."; - process.env.GIT_TERMINAL_PROMPT = "0"; - - // Clone the repository - const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit); - clonedRepoPath = join(cwd, repo_name); - - // Remove .git directory to avoid uploading git history - const gitDir = join(clonedRepoPath, ".git"); - if (existsSync(gitDir)) { - rmSync(gitDir, { recursive: true, force: true }); - console.log("Removed .git directory"); - } - - // Upload to S3 - const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`; - const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace); - - return { - success: true, - message: "Repository cloned and uploaded to S3 successfully", - s3_path: s3Path, - commit_hash: commitHash, - file_count: fileCount, - }; - - } catch (error) { - console.error("Error in git clone and upload:", error); - throw error; - } finally { - // Clean up cloned repository - if (clonedRepoPath && existsSync(clonedRepoPath)) { - rmSync(clonedRepoPath, { recursive: true, force: true }); - console.log("Cleaned up cloned repository"); - } - } -} - -async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise { - const sshIdFiles = await Promise.all( - git_ssh_identity.map(async (varPath, i) => { - const filePath = join(cwd, `./ssh_id_priv_${i}`); - - try { - // Get variable value using windmill - let content = await wmillclient.getVariable(varPath); - content += '\n'; - - // Write file with content - await fs_async.writeFile(filePath, content, { encoding: 'utf8' }); - - // Set file permissions to 0o600 (read/write for owner only) - await fs_async.chmod(filePath, 0o600); - - // Escape single quotes for shell command - const escapedPath = filePath.replace(/'/g, "'\\''"); - return ` -i '${escapedPath}'`; - } catch (error) { - console.error( - `Variable ${varPath} not found for git ssh identity: ${error}` - ); - return ''; - } - }) - ); - - const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`; - return gitSshCmd; -} - -async function git_clone( - cwd: string, - repo_resource: GitRepository, - commit?: string, -): Promise<{ repo_name: string; commitHash: string }> { - if (commit) { - return git_clone_at_commit(cwd, repo_resource, commit); - } else { - return git_clone_at_latest(cwd, repo_resource); - } -} - -async function git_clone_at_commit( - cwd: string, - repo_resource: GitRepository, - commit: string, -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const repoPath = join(cwd, repo_name); - await fs_async.mkdir(repoPath, { recursive: true }); - - process.chdir(repoPath); - - let args = ['init', '--quiet'] - if (branch) { - args.push(`--initial-branch=${branch}`) - } - await runCommand(undefined, 'git', ...args); - - await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url); - - await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit); - - await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD'); - - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function git_clone_at_latest( - cwd: string, - repo_resource: GitRepository -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - // Handle Azure DevOps token if needed - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const args = ["clone", "--quiet", "--depth", "1"]; - if (subfolder !== "") args.push("--sparse"); - if (branch !== "") args.push("--branch", branch); - args.push(repo_url, repo_name); - - await runCommand(-1, "git", ...args); - - const fullPath = join(cwd, repo_name); - process.chdir(fullPath); - - if (subfolder !== "") { - await runCommand(undefined, "git", "sparse-checkout", "add", subfolder); - const subfolderPath = join(fullPath, subfolder); - - if (!existsSync(subfolderPath)) { - throw new Error(`Subfolder ${subfolder} does not exist.`); - } - - process.chdir(subfolderPath); - } - - // Get the commit hash - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function uploadDirectoryToS3( - directoryPath: string, - s3BasePath: string, - workspace: string, -): Promise { - console.log(`Uploading ${directoryPath} -> ${s3BasePath}`); - - // Walk once into a flat task list so we can drive a bounded-concurrency pool. - const tasks: { localPath: string; s3Key: string }[] = []; - function walk(dir: string, s3Path: string) { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name); - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - walk(fullPath, s3Key); - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }); - } - } - } - walk(directoryPath, s3BasePath); - console.log(`Discovered ${tasks.length} files to upload`); - - let nextIndex = 0; - let uploaded = 0; - let lastReport = 0; - async function worker() { - while (true) { - const idx = nextIndex++; - if (idx >= tasks.length) return; - const { localPath, s3Key } = tasks[idx]; - const fileContent = fs.readFileSync(localPath); - const blob = new Blob([fileContent], { type: 'application/octet-stream' }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: s3Key, - requestBody: blob, - }); - uploaded++; - if (uploaded - lastReport >= 25 || uploaded === tasks.length) { - lastReport = uploaded; - console.log(`Uploaded ${uploaded} / ${tasks.length} files`); - } - } - } - await Promise.all( - Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker()) - ); - - // Marker is the LAST write — its presence is what the viewer checks for. - const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`; - const markerBody = JSON.stringify({ - completed_at: new Date().toISOString(), - file_count: tasks.length, - }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: markerKey, - requestBody: new Blob([markerBody], { type: 'application/json' }), - }); - console.log(`Wrote completion marker: ${markerKey}`); - - return tasks.length; -} - -function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise { - const nargs = secret_position != undefined ? args.slice() : args; - if (secret_position && secret_position < 0) - secret_position = nargs.length - 1 + secret_position; - - let secret: string | undefined = undefined; - if (secret_position != undefined) { - nargs[secret_position] = "***"; - secret = args[secret_position]; - } - console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`); - - return new Promise((resolve, reject) => { - const process = spawn(cmd, args); - - let stdout = ''; - let stderr = ''; - - process.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - process.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - process.on('error', (error) => { - let errorString = error.toString(); - if (secret) errorString = errorString.replace(secret, "***"); - console.log(`Shell command FAILED: ${cmd}`, errorString); - const e = new Error( - `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}` - ); - reject(e); - }); - - process.on('close', (code) => { - if (stdout.length > 0) { - console.log("Shell stdout:", stdout); - } - if (stderr.length > 0) { - console.log("Shell stderr:", stderr); - } - if (code === 0) { - console.log(`Shell command completed successfully: ${cmd}`); - resolve(stdout); - } else { - reject(new Error(`Command failed with code ${code}: ${stderr}`)); - } - }); - }); -} - -async function get_gh_app_token() { - const workspace = process.env["WM_WORKSPACE"]; - const jobToken = process.env["WM_TOKEN"]; - const baseUrl = - process.env["BASE_INTERNAL_URL"] ?? - process.env["BASE_URL"] ?? - "http://localhost:8000"; - const url = `${baseUrl}/api/w/${workspace}/github_app/token`; - - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${jobToken}`, - }, - body: JSON.stringify({ job_token: jobToken }), - }); - - if (!response.ok) { - const errorBody = await response.text().catch(() => ""); - throw new Error(`GitHub App token error (${response.status}): ${errorBody || response.statusText}`); - } - const data = await response.json(); - return data.token; -} - -function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { - const url = new URL(gitHubUrl); - return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`; -} diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index dc06a22fa0..6f7dc5871e 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -13,8 +13,8 @@ the dominant way dbt is orchestrated today. - **In**: run an unmodified dbt project synced into Windmill, one Windmill job per invocation, live per-model observability, dbt models as first-class assets in the existing asset graph. -- **Out**: one Windmill job per dbt model, `state:modified` / slim CI, - `dbt docs` hosting, semantic layer, dbt platform integration. +- **Out**: one Windmill job per dbt model, slim CI orchestration, `dbt docs` + hosting, semantic layer, dbt platform integration. - **CE**: the runtime, the manifest ingest, the asset graph and every piece of UI ship in CE, as do all adapters except two. Only the `mssql` and `oracle` adapters are EE, mirroring the native `ScriptLang` boundary (decision 21). @@ -35,8 +35,8 @@ the dominant way dbt is orchestrated today. | 10 | Private repo auth | Not applicable: the project is synced, not fetched | | 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | -| 13 | Manifest storage | Sidecar table for nodes/edges. Full manifest **not** stored — see below | -| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below | +| 13 | Manifest storage | Sidecar table for nodes/edges; the whole manifest is kept once per environment, for deferral — see below | +| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** and real column schemas come from the engine's parquet index, opt-in per project — see below | | 15 | Node rendering | Asset nodes per model plus one runnable node for the script | | 16 | Progress | Live, from the JSON event stream | | 17 | Test failures | Honor dbt's own `severity` | @@ -47,6 +47,8 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | +| 26 | Deferral | A durable state per environment, published by the runs whose relations are the script's; `defer` is a per-run toggle. See below | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -157,11 +159,15 @@ build and an enterprise build whose key did not verify. workspace warehouse's NAME, so two scripts running against the same warehouse agree on identity. -The SCHEME names the producer, because dbt is the only thing that creates one of -these: no other language derives warehouse relations, `// materialize` takes -DuckLake targets only, and a dbt run does not dispatch. Calling the kind -something generic promised a parity with native Snowflake and BigQuery scripts -that does not exist. +The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what +put warehouse relations in the asset graph and is what derives them from a +project; no other language *infers* one, and calling the kind something generic +promised a parity with native Snowflake and BigQuery scripts that does not exist. +A script can nonetheless DECLARE that it writes one — `// materialize manual +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -287,6 +293,32 @@ dbt hands the driver — it is written beside `profiles.yml` and pointed at by `sslrootcert`, as it is for a translated postgres resource. `profile.schema` and `threads` from the descriptor override their block keys rather than joining them. +**A `snowflake_oauth` warehouse behaves like a key-pair `snowflake` one**, +although its credential is an access token that lasts ten minutes: + +- The token goes under the `authenticator` each engine reads as an access + token: `oauth` on dbt-core 1.x, `jwt` on the Rust engines. Those read `oauth` + as the refresh-token flow and refuse a profile without client credentials, + while dbt-snowflake has `jwt` only from 1.9 and the 1.x engine can resolve 1.8. +- Every dbt process that logs in re-resolves the warehouse first + (`PreparedProject::refresh_profile`), and resolving an expired OAuth token + refreshes it. So the build, the `after_all` tests, a node retry and the + column-lineage pass each start with a live token, unless resolving fails: the + process then keeps the profile it already has, token included. +- Run identity masks credentials (`RenderedProfile::identity`), so a token + refreshed between a failure and its retry still matches. +- The OAuth connect flow asks for `database`, `warehouse`, `role` and `schema` + (`resource_fields` in `oauth_connect.json`). No token response carries them, + and dbt needs a database. + +One gap stays: a login after the token its dbt process started with has expired +fails. That is a thread whose first connection opens late in a long process, or +a process's first login when the token it was handed had only seconds left. +Refreshing tokens ahead of expiry would narrow it, but a token's lifetime is not +stored, so no margin fits every provider. Closing it would mean handing dbt the +refresh token and the instance's client secret, which any model can read on +dbt-core 1.x. + Three things follow, and they are the reason for the rule rather than consequences to work around. @@ -429,7 +461,9 @@ Two things make that safe rather than a widening: model set and its relations are already visible to them. - **`raw_code` is gated separately**, on an `EXISTS` against `script` in the authed transaction. The body of a model is the project's source code and stays - behind access to the project, whatever the shape query resolved. + behind access to the project, whatever the shape query resolved. `column_schema` + and the column trace behind `/jobs/dbt_column_lineage/{id}` take the same gate, + for the same reason: both are the shape of what the author wrote. The path and hash coming from the job row rather than the query also means a caller cannot pin one project's version while naming another's run. @@ -648,10 +682,13 @@ Two consequences worth knowing: dropped would be filtered out of its own run's graph. The pinned version's nodes are the scope instead. -## No cascade from dbt, and no pipeline membership +## No cascade *from* dbt, and no pipeline membership A finished dbt run does not trigger anything. Its models are recorded, drawn and -tracked; they do not fan out. +tracked; they do not fan out. The opposite direction does: a script that declares +`// materialize manual dbt:////` is an ordinary producer +of that relation, and its completion wakes `# on dbt://` subscribers +through the same fan-out every other asset kind uses. A dbt script is also not a pipeline member (`in_pipeline` is forced false for `ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like @@ -663,22 +700,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what puts a native script reading one of them on the same node, and it is independent of pipeline membership. -dbt already orders its own DAG, so a cascade would only ever add one thing: -waking a Windmill script that reads a mart. That edge is real but narrow, and -only half of it exists — nothing outside dbt can declare a `dbt://` write -(`// materialize` accepts DuckLake targets only), so the reverse direction, an -ingestion script waking a dbt project, cannot be expressed at all. - -Against that, dispatching correctly from dbt is not cheap. A run's `select` can -build any subset of the project, so the deploy-time write set is not what ran; -using it wakes consumers of relations the run never touched, and narrowing it -needs a per-job record of what was built, which the per-relation state table -cannot supply (it keeps one row per relation, stamped with the last writer). +dbt already orders its own DAG, so a cascade out of a run would only ever add one +thing: waking a Windmill script that reads a mart. That edge is real but narrow, +and dispatching it correctly is not cheap. A run's `select` can build any subset +of the project, so the deploy-time write set is not what ran; using it wakes +consumers of relations the run never touched, and narrowing it needs a per-job +record of what was built, which the per-relation state table cannot supply (it +keeps one row per relation, stamped with the last writer). So dbt materializes and reports, and `asset_dispatch` returns early for -`ScriptLang::Dbt`. A `# on dbt://` subscription is refused outright at -deploy rather than accepted and left dormant — an edge drawn on the canvas that -can never fire is worse than an error saying so. +`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should +notify — that decision is the work, not the plumbing. + +### Declaring the write, and which subscriptions are refused + +`// materialize manual dbt:////` is how an ingestion +script says it writes a warehouse relation. `manual` is not a mode but the only +mode: nothing generates warehouse DDL, so the script issues its own write and +Windmill records the outcome — the same `materialized_partition` row a DuckLake +target lands, so the relation carries a last writer on the run page and the graph. +It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path +(`record_declared_warehouse_write`) rather than in an executor, for the same +reason. Identity is unchanged — the physical relation — so the ingestion script +and the dbt model reading it are one node, and a `source` declared on the relation +puts the whole thing on one lineage. `// data_test` is refused beside it: those +checks are probes the DuckDB executor splices around a managed write, so on a +warehouse relation — which the script writes itself, from any language — nothing +would run them, and a declarer would deploy green with its assertions silently +skipped. Assert on the relation with a dbt test in the project that reads it. +The `` segment is resolved at +deploy for the same reason a descriptor's `profile.warehouse` is: a name no +warehouse answers to is not a namespace, it strands the write on a node nothing +else reaches. + +Known boundary, shared with every other runtime pipeline annotation: the record +is written from the normal execution path, and recording and cascading are decided +separately, so the routes off it differ. + +* A **dedicated worker** never enters that path — it bypasses the record exactly + as it bypasses `// partitioned` resolution — while its job is still a top-level + `Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It + cascades and records nothing, leaving the relation with no last writer. +* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which + `is_eligible_kind` rejects. Neither record nor cascade. +* A **flow step running a deployed script** enters the path as a `Script` job, so + it records — and carries a `flow_step_id`, so it never cascades. +* A **flow step with an inline body** is a `FlowScript` job, which the recording + guard excludes along with previews: neither. + +Fixing the recording half is one change for every runtime pipeline annotation, +not this one. + +A `# on dbt://` subscription is held to the same relation a producer +is — a whole `//` under a configured warehouse, checked +by the validator the `// materialize` target goes through, since two spellings of +that rule would refuse and accept the same string. Beyond that it is refused in +exactly one shape: when every script that writes that relation is a dbt one. Nothing +produces it yet is NOT that shape — a subscriber may be deployed before its +producer, as for every other asset kind, and refusing there would break +deploy-order-independent syncs. A dbt script may neither subscribe nor declare a +`// materialize`: its graph ingest republishes that path's trigger and asset rows +wholesale, so either annotation would deploy something the dependency job then +silently removes — while the declared write would still stamp the relation on +every run. + +The producer set is read as it stands committed, minus the deploying script's own +rows — those describe the version being replaced, so a script dropping its +`// materialize` while adding a subscription would otherwise count itself as the +producer that wakes it, which it could not be anyway (the dispatcher skips +self-loops). + +What that leaves is a subscription accepted while it was live and later orphaned. +A dbt project that claims the relation afterwards names those edges in its own log +rather than leaving them silently dormant — the same "an edge that can never fire +is worse than saying so" the refusal is for, at the other point where it is +knowable. Both points that publish ownership warn: the deploy, and a run whose +static descriptor found its profile moved. An agent run publishes none — it is +forced to per-run models, so it stores a job-pinned snapshot and leaves workspace +ownership with the deployed graph — so it cannot orphan a subscription either. + +Two orphanings are reported nowhere, and both are accepted rather than overlooked. +A native producer that drops its `// materialize` and leaves dbt alone on the +relation: the deploy that causes it does not touch the subscriber. And the +interleaving where a dbt ingest commits between a subscriber's producer check and +its own commit — the check sees no producer and accepts, the ingest's warning +query sees no trigger and says nothing. Closing the second means a per-relation +lock shared by the deploy path and the ingest, and the ingest takes +`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation +locks first inverts that order into a deadlock across two subsystems — a worse +failure than the cosmetic edge it would prevent. Both are bounded the same way: +the next deploy of that project warns, and the canvas is where they show +meanwhile. A plain READ still renders the consumer beside the model, which is what makes the lineage one graph — but it is written in the script's own code, not in a @@ -687,8 +801,8 @@ comment: the body parsers resolve an asset URI from a string literal Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the languages with a body-asset parser; the native warehouse ones (snowflake, bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart -they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a -selective run should notify — that decision is the work, not the plumbing. +they consume joins the graph only once that inference exists — while a relation +one of them WRITES joins it now, through the annotation. ## Live per-model progress, and why only dbt-core 1.x has it @@ -857,6 +971,8 @@ profile: select: ["tag:nightly+"] exclude: [] test_behavior: build # build | after_all | none +column_lineage: false # opt in to the static-analysis pass that + # produces column-level lineage (decision 14) vars: # typed: numbers/bools/lists keep their type, run_date: "{{ run_date }}" # and string leaves take job arguments strict: false @@ -1143,19 +1259,513 @@ block, since that is what `dbt_run_state` saves and `invocation_args` publishes. without failing. Overriding this would make the same project behave differently on Windmill than locally, breaking the core promise. +## Durable state per environment, and what defers to it + +`dbt --defer --state ` resolves a `ref()` the run does not build to the +relation the manifest in `` names, instead of to the schema this run writes +into. That is what lets one model be rebuilt into a scratch schema without +rebuilding everything above it, and it needs a manifest of the environment the +project actually lives in. + +Nothing that already existed could supply one. `dbt_run_state` answers a +different question — it holds the LAST run whatever its outcome, keyed by the +principal, so `dbt retry` can resume its failures — and the worker-local +generations behind it are a cache: the next run of a project usually lands on a +worker holding neither artifact. So the state is its own table, +`dbt_environment_state`, one row per (workspace, script path, environment), +holding `manifest.json` and `run_results.json` from the last SUCCESSFUL run. +Success is half of the contract: a relation a later run defers to has to exist. + +### The environment is the warehouse, the target and where they resolve to + +The workspace warehouse's name, the target dbt actually runs, and the database +and schema that target resolves to — the pair `relation_root` reports to the +graph's drift check. Each component is length-prefixed rather than joined on a +separator — `|||`, each written `:`, +so `main`/`prod`/`analytics`/`dbt_wh_defer` is stored as +`4:main|4:prod|9:analytics|12:dbt_wh_defer`. A target name and a schema are both +the user's own strings, so `prod|analytics` + `scratch` and `prod` + +`analytics|scratch` would otherwise be one key, and a profile moving between them +would read as the same environment rather than as one nothing has published. What +a message names is spelled out instead, never the encoded key. + +The target is the EFFECTIVE one, not the descriptor's `profile.target`: a +descriptor naming none inherits the workspace warehouse's, or the default in the +project's own `profiles.yml`, so reading the descriptor's would file every +inherited target under one empty name — and a `target.name` macro decides where a +model is built. + +The last two are in the key because deferring is resolving a relation NAME. A +warehouse repointed at another database, or a `profile.schema` moved by a +redeploy, keeps the first two while putting every relation somewhere else, and a +manifest is a list of relation names — there is no other way to notice. Keyed on +the first two alone, such a move would hand the next deferring run the names of +relations that are no longer there. Keyed on all four, it reads as an +environment nothing has published yet, which is what it is. + +What the key deliberately does NOT carry is the resolved connection. That is the +`profile_digest` a retry is held to. It masks credentials, but it still moves +with changes that move no relation, like another Snowflake warehouse or role; a +warehouse pointing somewhere else entirely is decision 11's accepted limitation, +spelled the same way here as everywhere else. + +Today one script has one environment, because a descriptor fixes both the +warehouse and the target and a run cannot override either. The key is what makes +the *later* item — fork and preview environments — an addition rather than a +migration, and what makes a profile move detectable now. + +### Which runs publish it + +A successful `build` that did not itself defer, and whose graph becomes what the +script owns (`GraphRefresh::publishes_ownership`) — the same condition as the +graph's and the same reason: an invocation that scoped its own model set — a +`vars` or `select` override, or a descriptor dynamic by construction — describes +where the CALLER put those relations, not where this project's models live. +Publishing it would point every later deferral at one caller's scratch schema. + +**A run that deferred never publishes, whatever narrowed it**, and that is a +separate condition rather than a consequence of the first. A deferring run built +some of the relations its manifest names and resolved the rest out of the state +it read, so recording that manifest would claim relations nothing built — and a +model renamed since would be recorded under a name only a full build creates, +breaking every later deferral until one repairs it. `publishes_ownership` cannot +see this: it reads the caller's overrides, and a descriptor that already narrows +`select` needs none. + +A `retry` publishes nothing. Its `run_results.json` names only the nodes it +redid, so the environment would come to claim a run of a handful of models. The +environment's state is therefore the last full successful build, exactly as dbt +Cloud's "last successful run" is, and a run recovered by a retry leaves it at +the previous one. + +The AUTOMATIC in-job node retry is the same artifact under a different name: a +build it recovers is a successful build, but the `run_results.json` on disk is +the retry's. Such a run publishes the manifest **without** results, rather than +with a set describing some other slice of the build — the manifest is a function +of the project rather than of what ran, so deferral is unaffected. A `result:` +selector is the one thing left with nothing to read, and it is refused by name +against such a publication rather than passed to dbt (see "Selectors that read +the state" below). + +Under `test_behavior: after_all` the stored `run_results.json` is the test +phase's, because that is what the second invocation leaves in the target +directory — the same artifact a local `dbt run && dbt test` leaves behind. + +**What that condition means for what the artifacts may carry**, and why this +table is keyed by environment where `dbt_run_state` is keyed by principal. dbt +records the invocation's flags into `run_results.json`, and Windmill resolves +`$var:` / `$res:` references before dbt sees them — which is exactly why the +retry state is per-principal, so one caller's resolved `select` and `vars` are +not restorable by the next. Here they cannot be one caller's: a publishing run +added nothing of its own, and a descriptor that interpolates a `{{ }}` +placeholder into `vars` never publishes at all, so what is recorded is the +descriptor's own arguments — the script's content, which anyone entitled to run +it may already read. Widen the publish condition and that stops being true. + +### Where the blob goes + +`run_results.json` is small; `manifest.json` is not, and grows with the project +(535 KB on a two-model fixture). Each takes the same two homes: inline in the row +under `DBT_STATE_INLINE_MAX_BYTES` (8 MiB), and the INSTANCE's object storage +above it, with the row keeping the key. Inline is what makes the feature work on +an instance that has configured no storage at all; the ceiling is what stops one +project's manifest from becoming a multi-megabyte row rewritten by every run. A +project past the ceiling with no storage configured is told so, in the job log, +naming the setting and the variable — the run itself still succeeds, since +losing the state costs the next deferral rather than the build that just ran. + +**The instance store, not the workspace's**, which is where every other internal +worker artifact already lives (bun bundles, python wheels, job logs, the global +cache). The workspace bucket is the one members read and write through +`job_helpers/*` and `wmill.write_s3_file` with a caller-supplied key, and only +`volumes/` is reserved there — so a manifest under it is one any member could +replace, and the next deferring run would hand dbt an attacker-chosen +`defer_relation` for every unbuilt `ref()` while holding the script's warehouse +credentials. Its compiled SQL would be readable there too, for a project the +reader may have no access to. The consequence to know: a project past the ceiling +needs the instance store configured, which is an EE feature, so on CE the ceiling +is the limit and `DBT_STATE_INLINE_MAX_BYTES` is how it moves. + +Each publication writes its OWN keys +(`wmill_dbt_state///./`) +and the row switches to them in one statement, so an upload never overwrites an +artifact the committed row still names: a run that fails between its two uploads, +or between them and its row, leaves the state pointing at the pair it already +had. The objects the commit displaced are dropped afterwards, never before, since +a reader that has already read the row is about to fetch them; a reader that +loses that race re-reads for as long as the row keeps MOVING, rather than +reporting a state that is there. A reader takes no lock, so successive +publications can each overtake one; an unmoved row whose objects are gone is the +error that means what it says, and a bound on the re-reads is the other, for a +project republishing faster than a run can read. What a publication uploaded and then could not commit is dropped on the +way out — except after a commit that REPORTED an error, where what was lost may +be only the acknowledgement: dropping then would leave a committed row naming +objects that are gone, so an orphan is the cheaper side to take. + +The path and the environment are only a prefix of that key. The row is what says +where an artifact is, which is why state can travel with a renamed script and go +on naming objects under the old path's digest. The rest of the key is the job and +a per-EXECUTION nonce — zombie recovery re-runs a job under its own id, so keyed +on that alone a second attempt would overwrite the objects the first attempt's +committed row still names, then read those keys back as displaced and drop them. + +Publishers of one environment serialize on `pg_advisory_xact_lock`, so only one +of them settles the row and the objects it displaces at a time — an advisory lock +rather than the row's, because the first publish of an environment has no row to +lock and is exactly when two runs of a newly deployed script are most likely to +race. + +### Retention + +None, deliberately, and this is where it differs from the graph tables next +door. Those are pruned by age by the dbt runs themselves because their reader is +a transient run page. This one holds a single row per script per environment, +replaced in place, so it does not grow with runs — and its reader is every later +run of that script, so a project that runs monthly must still find last month's +state. It goes with the script instead: a path no live dbt version occupies any +more clears it, alongside `dbt_run_state` (`clear_dbt_script_state`, +`clear_dbt_script_state_if_path_retired`). + +The write carries a guard of its own, and it names the VERSION rather than the +path: the live dbt script there must be the one this job ran, or a later version +of it (`hash = $n OR $n = ANY(parent_hashes)`). "Some live dbt script is here" — +which is what the retry state settles for — is also satisfied by a script created +at a path this one was renamed away from, and this job's manifest would then +become that project's deferral state. A preview names no version and so publishes +nothing, which is right for a run of content that was never deployed. + +The job's KIND is checked beside it, because a preview carries a caller-supplied +`script_hash` into `runnable_id` (`run_preview_script`): the version alone would +let anyone who may run a job publish arbitrary content as a deployed script's +state. A flow or app step naming a deployed dbt script by path is an ordinary +`script` job carrying that script's own hash, so it publishes like any other run; +only INLINE flow code is a `FlowScript`, and that has no deployed version to +publish for. + +That guard HOLDS the script row (`FOR SHARE`) for the rest of the publication, so +a rename, archive or delete of the path either waits for it or is seen by it. +Read unlocked, it leaves a window where the lifecycle clear finds no row to take, +finishes, and the publication then commits state at a path a new script goes on +to occupy. The script row is taken before the sidecar, which is the order every +other dbt writer takes and what keeps the two off a deadlock. + +An artifact too large for its row is left in the store when the row is cleared, +as a deleted script leaves its bundle: reaching it from the delete would mean an +object-store client in `windmill-common` and a delete that has to land after the +caller's transaction commits, for one object per environment of a script that is +gone. + +### Asking for it + +`defer` is a field on the `build` command block, defaulting to the descriptor's +own `defer:`. A per-run toggle rather than a descriptor-only setting, because the +run that publishes an environment's state and the run that defers to it are two +invocations of ONE script (decision 6: N scripts means N projects): a project +that could only defer by descriptor could never populate the state it reads. + +A project whose profile selects its schema or database with a TEMPLATE — either +delimiter, since dbt renders `{% … %}` blocks as well as `{{ … }}` — is refused a +deferral outright, and publishes no state either: dbt renders those and Windmill +does not, so two renderings resolve to one `relation_root`, and a +deferral after the value changed would resolve every unbuilt `ref()` through the +previous location's manifest. Both sides, because a published template would sit +under a key a literal profile shares, and de-templating later would make that +stale manifest readable as the new location's. It covers a project-owned +`profiles.yml`, a `dbt_profile` resource — one block of the user's own file, +copied through unchanged — and a `profile.schema` written as given. Plainly +absent is different: that is the adapter's default, which does not move. + +A run that asks to defer with nothing published is refused, naming the +environment and the runs that cannot publish one. The alternative — running +without deferral — fails deep inside dbt with a relation-not-found the caller has +no way to connect back to a missing state. An agent worker is refused the same +way and for a reason it can act on: it reaches the database only through the API, +which does not expose this table. + +A `show` defers too, and every engine takes the flags on it. It compiles the +model it previews, so a model whose upstream this environment built and this run +did not is exactly the case a deferral exists for. So does the `dbt ls` that +resolves what a run's selection owns, without which a `result:` selector — which +reads `run_results.json` out of the state directory, and which `select` passes to +dbt verbatim — would fail before the build that would have honoured it. + +The result carries `deferred_to`, the run whose state was used. Without it what +a deferring run built against is unrecoverable, since the next successful run of +that environment replaces the state. + +### Selectors that read the state, and why they are refused rather than passed + +`--state` also feeds dbt's own selector methods, so publishing the state is what +makes `state:modified+`, `state:new` and `result:error+` resolve at all. Only a +deferring run is handed the directory, so a `state:` or `result:` method in +`select` or `exclude` without `defer` is refused before dbt starts. + +Refused, rather than left to dbt, because the engines disagree about it and two +of the three disagree silently. Given a state selector and no `--state`, +dbt-core 1.x raises (`Got a state selector method, but no comparison manifest`, +exit 2), but dbt-sa-cli 2.x and fusion read a MISSING state as an EMPTY one and +exit 0: `state:modified` then selects nothing and the run reports success having +built nothing, while `state:new` selects everything, because against an empty +state every node is new. A scheduled run that quietly stops doing work, or +quietly rebuilds the project, is the failure this state exists to prevent. + +From the DESCRIPTOR they are refused whether or not the run defers, and the +message says so. That selection is also what decides which nodes the script owns, +and the deploy resolves it before any run exists, with no state to compare +against. "Whatever changed last" is not an ownership answer. They describe one +run, so they belong in a run's own `select`. + +`source_status:` is refused under any setting: it compares `sources.json`, which +`dbt source freshness` writes and no run publishes here, so there is nothing to +compare against even while deferring. + +Two more refusals follow from the same argument, that a selector with nothing to +read must say so rather than resolve to a silent answer: + +- A `result:` method while deferring to a state that carries **no** + `run_results.json`. Publishing that is deliberate — a build recovered by + automatic node retry stores the manifest alone, its results describing the + retried nodes rather than the build ("Which runs publish it") — so `defer` + being on is not enough to know the file is there. Answerable only once the + state is loaded, so it is checked right after, naming the run that published. +- Any of them on a `parse`. A parse resolves a selection to store the graph and + never defers, so `defer` would not hand it a state at any setting, and the + remedy the other refusal offers would lead nowhere. It says that instead. + +Matching nothing is then an ordinary outcome for these methods, and for no +others. `state:modified+` selects the empty set exactly when nothing changed +since the published state, which is the answer a CI run wants, so a selection +naming a `state:` or `result:` method may resolve to no nodes. Such a run scoped +its own selection, so what it stores is a snapshot of its own and never what the +script owns, and nothing is un-wired by the empty set. + +The exemption is by METHOD, not by who chose the selection. Exempting every +caller-chosen one would take a misspelled model name, which resolves to nothing +just as surely, and report it as a build that did its work. An ordinary selection +matching nothing stays refused, from a run as from the descriptor — from the +descriptor because that one also decides ownership. + +Only what `select` and `exclude` spell directly. A method reached through a +`selectors.yml` definition is named nowhere the worker reads, and dbt's own +behaviour — including the silent one — is what stands there. + +### `--state` is also a retry's own argument, and that is a trap + +`dbt retry` reads the run it RESUMES from `--state`. Handed the deferral's +directory it resumes the successful run stored there, finds nothing failed, and +reports a green retry having rebuilt nothing — silently, on dbt-core 1.x, which +warns and exits 0. + +dbt-core 1.x has `--defer-state`, the deferral-only half of the pair, so a retry +there passes that and leaves `--state` alone. The Rust engines do not have it, +and a run that deferred is refused a retry on them, before the build: the +alternative is rebuilding the failed nodes with every `ref()` resolving into the +schema this run writes into, which for the narrowed run a deferral exists to +serve means writing them somewhere they do not belong. The automatic in-job node +retry is dropped for the same reason and says so in the log. + +The state directory is passed RELATIVE (`wm_dbt_state`, beside `wm_target` in the +job directory). dbt records the invocation's flags into `run_results.json` and a +later `dbt retry` restores them, so an absolute path would name the job directory +of the run being resumed, which is gone by then. Relative, it resolves against +the project root — whichever job directory the retry landed in. + +Three engine facts found while wiring this up, all worth knowing before filing a +bug against the feature. `dbt retry` on dbt-core 2.x restores **neither** the +resumed invocation's `--vars` nor its deferral: it re-parses with the current +(empty) ones, so a retry of a run that overrode `vars` rebuilds into the +descriptor's schema rather than the run's. That is independent of deferral and +predates it; the refusal above stops the deferring case from being the way it is +discovered. `dbt show` on either Rust engine prints a bare JSON array where +dbt-core frames it as `{"node": …, "show": […]}`, which `run_show` is written +against — so a preview there fails to parse whether or not it defers, and the +deferral itself resolves correctly under it. And neither Rust engine reached +dbt's own service-backed State (`--manage-state`) on any run measured here, so no +flag is passed to disable it. + +Because `select` reaches dbt verbatim, a deferring run also has a `--state` +directory for `result:` selectors, which is why `run_results.json` is stored +beside the manifest rather than the manifest alone. + ## Two decisions the implementation narrowed -**Decision 13 — no S3 copy of the manifest.** The sidecar holds every field the -graph renders; nothing reads a stored `manifest.json`, so writing one to S3 -would be an unread copy of data that is already reproducible by redeploying (or, -for a dynamic descriptor, by the next run). Worth adding the day something needs the -parts the sidecar drops — compiled SQL, macro definitions — and not before. +**Decision 13 — the manifest is stored once per environment, not per version.** +The sidecar holds every field the graph renders, so a copy of `manifest.json` +bought the graph nothing: it is reproducible by redeploying, or for a dynamic +descriptor by the next run. Deferral is the reader that changed that — it +resolves an unbuilt `ref()` through a manifest, and one on worker-local disk +answers for a machine's history rather than for the environment. So exactly one +manifest is kept per (script, environment), replaced by each successful run, +rather than one per version (see "Durable state per environment" above). -**Decision 14 — column lineage is not available.** The decision assumed -`manifest.json` carries column-to-column edges; it does not, in either core -engine. What it does carry is declared column *descriptions*, which are -ingested. Real column lineage would need Fusion (which does static analysis) or -a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. +**Decision 14 — column lineage comes from the parquet index, not the manifest.** +`manifest.json` carries no column-to-column edges, in any engine, and its +`columns` are the ones an author declared in `schema.yml`. Both halves exist in a +different artifact: `dbt compile --static-analysis strict --write-index` writes +`target/index/`, and two of its tables are `dbt.column_lineage.parquet` +(`from_node_unique_id`, `from_column_name`, `to_node_unique_id`, +`to_column_name`, `lineage_kind`) and `dbt.node_columns.parquet` (every column of +every node, with its declared type, its inferred type and its description). + +Three measured properties decide the shape of the ingest. + +**Strict analysis is a stricter dialect.** `select no_such_column from +ref(...)` is `UnresolvedIdentifier (dbt0227)` and exit 1 under `strict`, and +compiles fine under `baseline` (the default). So this is a separate `dbt compile` +with its own `--target-path`, never a flag on the build, and it is opt-in per +project: `column_lineage: true` in the descriptor. Off, nothing changes. On, a +project that cannot be analyzed keeps exactly the graph it had. + +The pass is best-effort about everything that is ITS: a wrong engine, a rejected +analysis, a missing or unreadable artifact, an over-long output and outrunning its +own time budget all degrade to partial lineage or none, plus a line in the job +log saying which. It is not best-effort about the JOB: a cancellation or the job's +own deadline fail it, because swallowing those would let a run that blew its +timeout inside an optional annotation publish a graph and report success. That +split is why the two halves have separate error contracts — the compile owns the +job's semantics and may `Err`; nothing the artifact does or fails to do is a +reason to fail a job, so an absent, unreadable or partial index is a value. The +decode still runs under the job poller, which both heartbeats through it and +ends it if the job is cancelled or completed meanwhile: the job reaching in, not +the artifact reaching out. The +budget is half the job's remaining wall clock, spent on the compile alone, so the +build that follows cannot be starved by it. + +**A failed pass still writes the index**, holding every edge of the models that +did analyze, so the artifact is read whatever the exit status and partial lineage +is a normal outcome. An unreachable *source* is milder still: `RemoteError +(dbt1014)` downgrades that model to `static_analysis: off` and the compile +succeeds. (Strict analysis queries the warehouse catalog for source schemas; a +`ref()`ed model is inferred statically and needs no built table.) + +**The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 — the version +`DBT_CORE_2X_VERSION` pins — accepts `--write-index` and `--write-lineage`, and +its own `views.sql` declares views over both tables, but it writes neither +parquet; only Fusion does today. The ADAPTER decides too: an experimental one +(postgres under `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`) turns static analysis off and +says so only in a warning on an otherwise successful compile. The gate is +therefore "the engine has the flag" (everything but 1.x, whose Python CLI has no +such option) plus "the file appeared", so a later release picking the feature up +needs no change here — and the job log carries the engine's own stderr whenever +no index appears, since without it "no column lineage" has no explanation. + +`lineage_kind` is stored as TEXT, not an enum. Three values exist — `copy` +(passthrough), `mod` (transformed) and `scan` (the column was read to produce the +ROW rather than the value: a join key, a `where` predicate, a `group by`) — and +the engine's own reader maps those three and passes anything else through, so the +set is the engine's to extend. All three are stored, and `copy`/`mod` are kept +first when the bound bites: a `scan` edge reaches every output column of its +model, so it is most of what a project's index holds and would draw as a complete +bipartite graph. Keeping it in the table anyway is what lets a later "show +indirect" view ask for it without every project being redeployed. + +Storage mirrors `dbt_edge` exactly: `dbt_column_edge`, keyed by (path, version, +job) with the same composite foreign key to `script`, so a version's column +lineage dies with the version and a run's snapshot with the sweep. + +**A table of its own, not `dbt_edge.column_lineage` JSONB.** Hanging the links on +the `ref()` edge they sit beneath would inherit its clone, prune, clear and +cascade paths for free, and it does not work: a model reading `{{ this }}` gets +column lineage from itself to itself, and `parent_map` has no self-loop, because +a model does not `ref()` itself. Those pairs have no `dbt_edge` row to attach to. +The loss is not hypothetical — an incremental that selects from `{{ this }}` +(`coalesce(p.dbl, s.dbl)`, `p.up as prev_up`) yields `up → prev_up` with kind +`copy`, a drawn edge meaning "this column carries the previous run's value". +Inventing self-loop `dbt_edge` rows to hold it is not an option either: that +table is `ref()` lineage. The typed column list lands in +`dbt_node.column_schema`, beside `columns` rather than merged into it — +`columns` stays what the author *declared*. + +Two things are user-visible. `column_schema` — every column of a relation, typed +and in the order the model emits them — rides the asset graph the details pane +already fetches, and replaces a panel that could only list the columns an author +happened to document. The edges are served by an endpoint of their own, +`assets/column_lineage`, which the pane asks for the selection it is drawing. + +Both are gated on being able to read the producing project, like the model's SQL: +a column-level view is the shape of what the author wrote, one level finer than +the `ref()` graph, which is ungated only because it draws relations the caller +already sees. A share-link viewer entitled to a dbt run therefore gets its +relations and `ref()` edges, and neither the SQL nor the columns. + +**One request per selection, and the gate re-decided per project.** The endpoint +takes every relation the view has reached and answers their union, because one +selection reaches several — a script's output column can derive from columns of +several models. Holding partial answers between selections instead was tried and +is what a client cache is: it produced a wrong premise for a relation two projects +describe, then staleness on redeploy, then a lost retry. + +The answer is the connected component around those relations, and that component +does not stop at the project that owns them: a relation one project produces is +another's source, so the walk resolves owners, reads their edges, walks, and +repeats for the relations that walk newly reached. Resolving once — for the +relations asked about — stops the trace at the first project boundary. The +security half is that a project reached this way is a project the caller may not +be entitled to, so the scope filter and the project's visibility are re-applied to +every project the expansion discovers, not decided once for the first owner set. + +A PINNED answer is the exception and needs none of it, whether it names a job or +a deployed version: the pin says which stored graph is on screen, and another +project's live graph is not part of it, so it answers for that one project. The +dbt editor pins by version on every selection and the run page pins by job; the +pipeline page pins nothing, and is where the walk crosses projects. + +**The component is bounded, and says when it was cut.** The renderer draws a box +per column, so a component past a few thousand edges is unreadable however it is +served — a synthetic 3000-model project whose models share a column returns 58k +direct edges and 7.3MB. The walk is breadth-first from the asked-for relations and +stops at 5000 edges, so what survives is the part nearest the selection rather +than an arbitrary slice, and `truncated` says so: a trace that stops short is +otherwise indistinguishable from one that ends. + +`truncated` is set only where a project this caller may read is left unread, or +where the walk itself was cut. Neither the expansion's size budget nor a relation +whose owners were never asked about is evidence on its own: a big project's small +component is answered whole, and the relations still waiting to be asked about at +the end are almost always owned by the project already in hand. The order the +rounds run in is what makes that decidable — the owners query comes before the +budget check, so the stop happens with a named unread project rather than a +suspicion of one. + +The FETCH is not bounded the way the answer is: a project's edges arrive whole, +because the walk is what decides which of them are in the component, and a +`LIMIT` would cut a set that need not contain the asked-for relation at all. The +seeds' own projects are therefore read whatever their size — reading them is the +answer. What bounds a single fetch is the ingest's own cap per version; the +budget bounds only the expansion on top of it. + +The walk is in Rust rather than a recursive CTE. `EXPLAIN ANALYZE` on that same +project measured 1243ms against 59ms for the query alone: a CTE has no index to +walk, so the recursive term rescans the doubled edge set once per level (11.7M +rows), while the same walk over a map is microseconds. + +The two halves of a trace — dbt's and the pipeline's — meet at shared node ids. A +DuckDB script's `// column x <- dbt://wh/s/model.col` mints the same +`(dbt, path, column)` node dbt's own lineage does, so the producer graph the asset +graph already carries and the dbt graph are MERGED rather than chosen between, and +a trace crosses that boundary in either direction. What the browser cannot close +in one request is a relation the server discovers whose columns are consumed by a +script that writes into a third project: the producer half of that hop is the +canvas's, not the server's, and the seeds were computed before the answer arrived. + +**The analysis pass takes the build's own `--full-refresh`.** `is_incremental()` +branches on it, so an incremental model reading `{{ this }}` compiles its +self-join — and any `ref()` inside that branch — only when the flag is absent. A +pass that used the descriptor's default while the run overrode it would store +lineage for SQL that run never executed. For the same reason an invocation that +overrides the flag counts as `per_run_models`: its graph is its own, keyed to the +job, rather than standing as the version's. + +That flag is not the whole of it, and the rest is a property rather than a bug to +fix. `is_incremental()` is also false when the target table does not exist, so an +incremental model has **two shapes and one ingest holds one of them**: a deploy +before the first build compiles the cold shape, and the same project deployed +again once its tables exist compiles the incremental one. A static descriptor +re-ingests on neither runs nor time, so what is stored stays whatever the compile +in front of it saw. dbt has no mode that emits both, and re-analyzing per run +would buy a second `dbt compile` on every build to keep a graph nobody asked to +refresh. The contract is therefore the honest one: a version's graph describes +the compile that produced it, and a run that re-ingests describes its own run. ## Concept mapping @@ -1167,7 +1777,9 @@ a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. | `materialized: incremental` | `append` or `merge` (by `unique_key`) | same | | `{% snapshot %}` | `scd2` | same, incl. `_current` handling | | `unique`/`not_null`/`accepted_values`/`relationships` | `data_tests` | exact 1:1 with the four `// data_test` kinds | -| declared column metadata | `columns` on the asset node | descriptions only; see the note below | +| declared column metadata | `columns` on the asset node | descriptions only, from the manifest | +| analyzed column schema | `column_schema` on the asset node | `dbt.node_columns.parquet`, opt-in | +| column-to-column lineage | `dbt_column_edge` rows, drawn as a column trace | `dbt.column_lineage.parquet`, opt-in | | model `tags` | node badge | `tag` | | source freshness | `freshness` | `last_success_at` chip | | `run_results.json` | materialization records | `record_materialization` | @@ -1195,7 +1807,10 @@ render through the existing `RunnableNode.svelte` / `AssetNode.svelte` / on the canvas mid-run. `record_materialization` per model. Profile and select pickers in the editor. Per-model failure triage in the run view. -**Phase 4 (not in this PR).** `--defer` and `state:modified`. Partition and +**Phase 4 (not in this PR).** Slim CI: the fork and preview environments a +deferral would name instead of its own. The selectors themselves are here, since +`state:` and `result:` read the published state like any deferral does; what is +missing is a per-branch environment to compare a CI run against. Partition and backfill integration so `BackfillRangeDialog.svelte` works on dbt models. `wmill dbt import ` reading `DbtDag(...)` kwargs. @@ -1215,18 +1830,33 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: script reading one of the marts gets an edge to it. 6. **Shared node**: a native script that READS a mart renders as a reader of the same node the dbt model writes — one node, not two islands. Declared with a - plain read (`# dbt://`), never `# on`: a `dbt://` subscription is - refused at deploy, because nothing but dbt writes a warehouse relation and a - dbt run does not dispatch (see "no cascade from dbt"). -7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each + plain read (`# dbt://`), never `# on`: a subscription to a relation dbt + alone builds is refused at deploy, since a dbt run does not dispatch. +7. **Declared write**: a native `// materialize manual dbt://` script + and a dbt project reading that relation as a `source` render as one node; a + run of the script records its materialization and wakes a + `# on dbt://` subscriber — a subscription only that producer makes + wakeable, the dbt project reading the relation being no producer of it (see + "no cascade *from* dbt"). +8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each build only the expected subset. -8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph +9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph from the run's own manifest, so a model that placeholder enables appears in the same run that builds it. -9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's +10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's own `profiles.yml` with env-var injection. -10. **Caching**: a second run reuses the cached `dbt_packages/` with no network +11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. +12. **Deferral**: a full run publishes the environment's state; a second run + that builds one downstream model into another schema resolves its unbuilt + `ref()` to the relation the state names, where the same run without `defer` + fails with relation-not-found. +13. **State selectors**: with a state published, `state:modified+` selects + nothing while the project is unchanged and exactly the changed model and its + children after one is edited. Without `defer` it is refused rather than + passed, and a `result:` selector against a state published by a + node-retry-recovered build is refused too, that one carrying no + `run_results.json`. Keep only tests that pin behavior a future change could break. Per AGENTS.md, delete development scaffolding before marking the PR ready. diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 32fff7ca23..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion (EL)"; this section records only what future feature work must not break. -- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and - `manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK +- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected + elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** — diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index f5ce2357ca..ae62d46b94 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 42 registered actions across seventeen features (`ai_session`, `ai_chat`, +It currently carries 51 registered actions across nineteen features (`ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, -`flow_step`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`, -`sso_groups_claim`). Nearly all of the +`flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, +`usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/docs/git-repo-viewer-hub-script.md b/docs/git-repo-viewer-hub-script.md index 7bfbe8160f..a1b5ea1482 100644 --- a/docs/git-repo-viewer-hub-script.md +++ b/docs/git-repo-viewer-hub-script.md @@ -1,7 +1,10 @@ # Git repo viewer — hub script -The hub script `clone_repo_and_upload_to_instance_storage` is published from -`windmill-integrations` and pinned in `frontend/src/lib/hubPaths.json` as +The hub script `clone_repo_and_upload_to_instance_storage` +([hub page](https://hub.windmill.dev/scripts/windmill/13968)) is published from +`windmill-integrations` +(`hub/windmill/scripts/action/13968_clone_repo_and_upload_to_instance_storage/script.ts`) +and pinned in `frontend/src/lib/hubPaths.json` as `cloneRepoToS3forGitRepoViewer`. Hub paths are exact version pins, so editing the script means publishing a new version and repointing that entry. @@ -23,6 +26,11 @@ The repo viewer in the Windmill app expects the hub script to: 3. **Write a completion marker** as the very last action of a successful run, so the API and frontend can distinguish a fully-populated S3 directory from a partial / interrupted upload. +4. **Follow symlinks that stay inside the checkout.** Both the git clone and + the archive extraction keep a repository's symlinks as links, and + `Dirent.isFile()` / `isDirectory()` are both false for a link, so a walk + that only checks those drops every linked file and directory from the + viewer. See [Symlinks](#symlinks). The marker file the frontend looks for is `.windmill_clone_complete` at the root of the per-commit directory: @@ -43,28 +51,74 @@ after the walk completes: ```ts const UPLOAD_CONCURRENCY = 16 const CLONE_MARKER_FILE = ".windmill_clone_complete" +const MAX_SYMLINKED_ENTRIES = 20_000 +const MAX_SYMLINKED_BYTES = 512 * 1024 * 1024 async function uploadDirectoryToS3( directoryPath: string, s3BasePath: string, workspace: string, -) { +): Promise { console.log(`Uploading ${directoryPath} -> ${s3BasePath}`) // Walk the directory once, producing a flat list of (localPath, s3Key) pairs. const tasks: { localPath: string; s3Key: string }[] = [] - function walk(dir: string, s3Path: string) { + const root = fs.realpathSync(directoryPath) + // Real paths of the directories being descended through. + const ancestors = new Set() + // What entries reached through a link have cost so far; see Symlinks below. + let symlinkedEntries = 0 + let symlinkedBytes = 0 + let symlinkBudgetSpent = false + function chargeSymlinkBudget(relPath: string, entries: number, bytes: number): boolean { + if (symlinkBudgetSpent) return false + symlinkedEntries += entries + symlinkedBytes += bytes + if (symlinkedEntries <= MAX_SYMLINKED_ENTRIES && symlinkedBytes <= MAX_SYMLINKED_BYTES) { + return true + } + symlinkBudgetSpent = true + console.log( + `Skipping ${relPath} and every symlinked entry after it: symlinks reach more than ` + + `${MAX_SYMLINKED_ENTRIES} entries or ${MAX_SYMLINKED_BYTES / 2 ** 20} MiB` + ) + return false + } + function walk(dir: string, relDir: string, viaLink: boolean) { + ancestors.add(dir) for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name) - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name - if (entry.isDirectory()) { - walk(fullPath, s3Key) - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }) + const relPath = relDir ? `${relDir}/${entry.name}` : entry.name + const linked = viaLink || entry.isSymbolicLink() + if (linked && !chargeSymlinkBudget(relPath, 1, 0)) continue + let localPath = join(dir, entry.name) + if (entry.isSymbolicLink()) { + const link = fs.readlinkSync(localPath) + try { + localPath = fs.realpathSync(localPath) + } catch (e: any) { + console.log(`Skipping symlink ${relPath} -> ${link}: cannot resolve target (${e.code})`) + continue + } + if (localPath !== root && !localPath.startsWith(root + sep)) { + console.log(`Skipping symlink ${relPath} -> ${link}: target is outside the repository`) + continue + } + } + const stat = fs.statSync(localPath) + if (stat.isDirectory() && ancestors.has(localPath)) { + console.log(`Skipping ${relPath}: links back to a directory it is inside`) + continue + } + if (linked && stat.isFile() && !chargeSymlinkBudget(relPath, 0, stat.size)) continue + if (stat.isDirectory()) { + walk(localPath, relPath, linked) + } else if (stat.isFile()) { + tasks.push({ localPath, s3Key: `${s3BasePath}/${relPath}` }) } } + ancestors.delete(dir) } - walk(directoryPath, s3BasePath) + walk(root, "", false) console.log(`Discovered ${tasks.length} files to upload`) @@ -114,9 +168,40 @@ async function uploadDirectoryToS3( requestBody: new Blob([markerBody], { type: "application/json" }), }) console.log(`Wrote completion marker: ${markerKey}`) + + return tasks.length } ``` +## Symlinks + +A link is resolved with `realpathSync` and followed only when its target lies +inside the checkout's real path. A file target is uploaded under the link's own +path; a directory target is walked as if it sat there, so +`inventories/prod/group_vars -> ../../shared/group_vars` shows up in the viewer +with its files. Everything else is skipped and logged: + +- **A target outside the checkout.** The repository chooses the target, and the + checkout sits in the job's working directory next to the ssh key + `get_git_ssh_cmd` writes (`../ssh_id_priv_0`) and the job's `args.json`. A + link to one of those, or to `/proc/self/environ` with the caller's + `WM_TOKEN`, would put it in storage for every reader of the resource. This + is why the walk does not follow links the way `aws s3 sync` does. +- **A target that cannot be resolved**: a dangling link, or a link loop + (`ELOOP`). +- **A directory that is already being walked higher up** (`loop -> .`, + `up -> ..`). The guard holds the real paths of the current descent only, as + `find -L` does, not every directory seen so far: a directory reachable + through two links is uploaded under both paths, as the checkout presents it. +- **Anything reached through a link once the budget is spent.** Because a + directory can be reached along many paths, two links to the next directory + at each level double the tree, and a repository a few dozen links deep would + expand past what the job can hold in memory. Every entry reached through a + link counts against a budget of 20,000 entries and 512 MiB. It is charged + before the link is resolved, so links that end up skipped count too, and + neither their work nor their log lines can multiply. Past the budget, the rest + are skipped with one log line. The checkout's own files are always uploaded. + ## Notes for review - **Concurrency level**: 16 is a starting point; tune based on instance @@ -127,6 +212,12 @@ async function uploadDirectoryToS3( paths on retry, so a partial upload + retry naturally heals. Old commit directories from before this patch are unreachable through the UI but still consume storage; an instance admin can prune them manually if desired. +- **A new pin doesn't refresh commits already uploaded**: the viewer keys + storage on the commit hash (`gitrepos/{workspace}/{resource_path}/{commit_hash}/`) + and only checks that the marker exists. So a commit uploaded by an earlier + script version keeps that version's tree (hub/28905's had no symlinks) until + the repository's head moves to a new commit, or an admin deletes that + commit's directory. - **Error propagation**: keep the existing `try/catch` in `main` so an upload failure surfaces in the job result and is shown in the new viewer error banner. diff --git a/docs/git-sync-gitlab-setup.md b/docs/git-sync-gitlab-setup.md new file mode 100644 index 0000000000..6e736dee37 --- /dev/null +++ b/docs/git-sync-gitlab-setup.md @@ -0,0 +1,170 @@ +# Git sync with GitLab + +GitLab has no equivalent of a GitHub App, so there is nothing to install and no +consent screen. What Windmill needs instead is one credential you create in +GitLab and paste once. With it, a GitLab repository gets the same managed +features an app-backed GitHub repository has: instant pull over a webhook, merge +requests opened on deploy, and a diff preview posted onto the merge request. + +## The credential + +Create a **project access token** on the project you are syncing (Settings → +Access tokens). It is a bot identity that outlives the person who created it, +which is what you want for a credential the instance uses unattended, and it +reaches exactly the one project. + +| | | +| --- | --- | +| Scope | `api` | +| Role | Developer to push deploy branches; **Maintainer** to also manage the webhook and open merge requests | +| Expiry | Required. A group service account PAT can be non-expiring on self-managed (see below); an access token cannot | + +**Use a separate token per repository.** A group access token works too and +reaches every project in the group, which is convenient for a lot of +repositories — but Windmill stores the credential per repository, and renewal +rewrites the repository it renewed for. Any other repository holding that same +token keeps the revoked one and stops syncing until you paste a new token there. +Each stranded repository says so on its card, so it is visible rather than +silent, but a token per repository avoids it entirely. + +`api` is a superset: it authorizes Git over HTTPS as well, so no separate +`write_repository` is needed to clone and push, and it is also what makes the +token renewable. A `write_repository`-only token can still push, but Windmill +cannot inspect or renew it and reports that in the workspace's git sync settings. + +### The identity Windmill acts as + +GitLab issues an access token to a bot user it creates for it — `project__bot_…` +for a project token, `group__bot_…` for a group one — and the bot's display +name is **the name you gave the token**. That name is the byline on everything +Windmill does: the author of deploy commits, of the merge requests it opens, and +of the preview notes it writes. Name it for what it is, `windmill-sync` or +similar, rather than something only you will recognise. + +Each token you create adds another bot member to the project or group. Renewal +does not — it keeps the same bot — so a repository accumulates one bot, not one +per year. + +Renewal goes through GitLab's own self-rotation endpoint. Both kinds of access +token are held as their bot user's personal access token, so the token rotates +itself and Windmill never needs a credential with rights over the project or +group. + +## Connecting a repository + +In the resource form for a `git_repository` resource, use the **GitLab** button: +paste the instance URL and the token, pick a project from the list, and Windmill +keeps the token for you. The resource itself gets the plain remote URL +(`"url": "https://gitlab.com/group/project.git"`), with no credential in it. + +The token is stored encrypted on the workspace, keyed by the repository it was +issued for rather than by the resource naming it, the same way a GitHub App +installation is held against the account it covers. Nothing reads it back out +over the API: the server attaches it when it talks to GitLab, and a sync job +receives it only against its own job token. Repointing a resource's `url` asks +for a different repository's token and finds none, so the edit carries nothing +with it; a repository that genuinely moved needs its token entered again. + +Because the repository is the key, the token is stored the moment you pick the +project, before the resource is saved. Renaming the resource later keeps it, and +cancelling the edit leaves a stored token that nothing uses until some resource +points at that repository again. + +Forks of the workspace read this one copy rather than getting their own, so +renewal reaches all of them at once and the token is not duplicated into every +descendant workspace. + +Treat workspace admin as equivalent to holding the token. An admin of the +workspace, or of any fork below it, can point a repository at a sync script they +wrote and have that job request the credential, exactly as they can for a GitHub +App installation token. Storing it this way keeps it out of the variables API and +out of every fork's own storage; it is not a boundary against the admins of those +workspaces. + +A URL with the token written into it, in the resource or in a secret variable +the resource points at (`"url": "$var:..."`), is a plain git remote: it syncs on +deploy and by polling, and nothing else. Windmill does not know the token is +there, so it registers no webhook, opens no merge request, and neither reports +nor renews its expiry. Use the **GitLab** button to hand the token to Windmill +if you want any of that. + +## Expiry and renewal + +Windmill reads `expires_at` from the token it holds and shows it on the +repository in the workspace's git sync settings. Within three weeks of expiry it +rotates the token through GitLab's own +`POST /personal_access_tokens/self/rotate`, stores the replacement, and verifies +it. Only the token can rotate itself, so one without `api` (or `self_rotate`) is +a permanent warning rather than something Windmill can fix. + +Only the workspace that holds a credential renews it, so one renewal serves the +whole fork chain instead of each fork racing to renew the same token. A fork +reads the parent's without holding one, so it never renews; the parent does, and +every fork sees the replacement at once. + +Rotation is deliberately never retried. GitLab revokes the old token the instant +it issues the replacement, and presenting an already-rotated token to `/rotate` +again is treated as reuse: it revokes **the whole token family, including the +live replacement**. So a rotation that succeeded at GitLab but failed to persist +is surfaced as an error to act on, not retried. + +Non-expiring tokens are possible only for a **group service account PAT** on +self-managed, with `require_personal_access_token_expiry` turned off in the +instance's application settings. A group access token is always rejected without +an `expires_at`. + +## What each managed feature needs + +| Feature | Needs | +| --- | --- | +| Instant pull | A project hook Windmill creates, so Maintainer; and a Windmill base URL GitLab can reach | +| Merge requests on deploy | Developer, plus the `api` scope | +| Diff preview on a merge request | The project hook, plus permission to post merge request notes | + +Instant pull falls back to checking the tracked branch about every minute when +the hook cannot be created or delivered, so nothing silently stops syncing. + +## Self-managed differences + +**Webhooks to a private network are blocked by default.** GitLab refuses to +create a hook pointing at a private or local address until an administrator +enables *Allow requests to the local network from webhooks and integrations* +(Admin → Settings → Network → Outbound requests, +`allow_local_requests_from_web_hooks_and_services`). A Windmill instance on the +same private network as GitLab needs this; without it, hook creation fails with a +"blocked" error and the repository keeps polling. + +**A relative-URL install is not supported.** GitLab can be served under a path +prefix (`https://example.com/gitlab`), and that prefix cannot be told apart from +a group of the same name: `example.com/a/b/c.git` is either group `a/b` project +`c`, or prefix `a` with group `b` project `c`. Windmill reads it as the nested +group, so on a relative-URL install it derives the wrong API base and the managed +features stay unavailable. Such a repository still syncs through its token URL, +which needs no API base. + +Everything else is identical: Windmill talks to `/api/v4` and needs +no inbound access of its own beyond the hook deliveries. + +## The deploy preview is a note, not a pipeline status + +On GitHub the preview is a check run: its own object, advisory unless the +repository makes it required. GitLab has no equivalent. Its only comparable +primitive is a commit status, and posting one has side effects Windmill will not +impose on a project: + +- GitLab files the status **as a job inside whatever pipeline already covers that + commit**, so a failed Windmill status fails the project's own pipeline, and its + reviewers see their test suite as failed. +- `allow_failure` is ignored on the commit-status endpoint, so the status cannot + be made advisory. +- On a commit with no pipeline it creates an `external` pipeline instead, which + then gates merging under *Pipelines must succeed*, including while it is still + running. + +So on GitLab the preview lives entirely in a **merge request note** that Windmill +keeps up to date: it carries the workspace, the status line, the commit, a link +to the job, and the full list of changes merging would deploy. A note cannot +block a merge or change what the project's own CI reports. + +The note is upserted rather than appended, so a merge request accumulates one +Windmill comment however many times it is pushed to. diff --git a/docs/git-sync-pull-design.md b/docs/git-sync-pull-design.md index 8961b0f047..53d36c82fe 100644 --- a/docs/git-sync-pull-design.md +++ b/docs/git-sync-pull-design.md @@ -186,6 +186,35 @@ Routing — an event/poll result is `(repo, ref, head_sha, sender)`: filters): fan out, each workspace pulls with its own filters; `wmill.yaml` in the repo stays authoritative for include/exclude. +Identity — a pull applies changes as a real workspace admin, never a reserved identity. +The schedules, triggers and app policies it deploys persist their deployer as the identity +they run as, and `validate_on_behalf_of` refuses reserved sentinels there, so a real admin +is what keeps those deployable and revocable (demote or remove the admin and what runs +under them stops). + +- The admin is `auto_pull.enabled_by`, stamped server-side with the email of whoever last + saved the git sync settings with auto pull on. Re-saving as another admin rotates it. +- A stamp naming someone who is no longer an active admin (demoted, or deactivated in the + workspace or on the instance), and not an active instance superadmin either, fails the + pull rather than falling back to someone else. A superadmin who is not a member runs it + under their instance username, and only while no member of the workspace holds that + username: `u/` resolves through the workspace's members before the email. A + repository whose settings predate the stamp runs as the workspace's first active admin + until they are saved again. +- The identity is resolved before the deploy check is posted, and a failure to resolve it + or to enqueue is recorded on the repository's status, not returned: a returned error + would fail the webhook delivery, and hosts disable hooks whose deliveries keep failing. + The next push or poll retries. +- Fork pulls run as the parent repository's identity, stamped or not, resolved in the + parent (revoking that admin there stops fork pulls too), and first add that admin to + the fork as an admin member, since a plain fork carries only its creator. The fork's + owner cannot be the identity: a non-admin's `wmill sync push` diffs against what it can + see, so an item in a folder it cannot read reads as a create and the push fails on every + commit. CI tests do run as the owner (Phase 7), because they only execute. +- Known and accepted: repo writers control the pull's includes through `wmill.yaml`, so a + fork's owner can commit a user file that makes them admin of the fork and read the + parent secrets it cloned, as with the `push-on-merge-to-forks` Action this replaces. + Loop prevention (pull → deploys → deployment callback → commit → push event): 1. Skip events whose sender is the app bot (`windmill-sync-helper[bot]` / @@ -618,6 +647,93 @@ repo's **Environments** timeline ("Production → Deployed"). Needs opt-in / later. The check-run version is the cheap default and matches the visual Cloudflare parity without a new permission. +### Phase 7 — CI test results check (WIN-2051) — implemented + +Surfaces Windmill's own CI tests (the `// test: script/...` annotation) as a +**"Windmill CI tests"** check run on **any PR** against the tracked branch, so a customer +can mark it a **required status check** and have Windmill CI results gate the PR — +replacing the documented GitHub Action that polls `ci_test_results_batch`. GitHub App-backed +only; reuses the Phase 4 `Checks: write` grant, so no new permission. Token repos keep the +Action, and GitLab merge requests get no CI-test surface for the same reason the Phase 4 +preview lives in a note there (a commit status would fail the project's own pipeline). + +Driven by the **`pull_request` webhook** — the same event Phase 4 already reacts to — +rather than the deploy push/pull, so it's uniform across how the PR's commit came to exist +(a fork deploy that pushes `wm-fork/**` and opens the PR, or an external push that gets +pulled in). CI tests run as separate async `ci_test` jobs in the **fork workspace** the PR +corresponds to; the check reflects that fork's current results on the PR head. + +- **State** — `git_sync_ci_test_check(workspace_id, repo_resource_path, head_sha)` (new + table). `workspace_id` is the **fork** whose `ci_test` jobs the check reflects; + `repo_resource_path` the repository (a fork can sync several, and two can hold the same + commit); `poster_workspace_id` is the **parent** whose GitHub-App installation posts the + run (the workspace that received the webhook and owns the repo hook). Plus `repo_url`, + `head_ref`, `check_run_id` (NULL until the create succeeds, and reset to NULL by a re-fired event + for the same head: the row is written first so a create that never gets recorded cannot + strand an in-progress run, and the poller retries any row without an id), `created_at`, + `concluded`, `conclusion`, `concluded_at`, `github_posted`. + Partial index `(workspace_id) WHERE NOT concluded OR NOT github_posted` (the live set the + hook + poller scan). +- **Open** — in the `pull_request` handler (opened/synchronize/reopened, or edited with a + base change, base = tracked): when the head lives in the base repo, resolve the fork + workspace from the head ref (reusing the fork-branch routing; + `resolve_pr_head_workspace`), persist the intent row with a null check-run id, then + `create_check_run` in_progress on `head_sha` via the parent's installation and adopt the + id (the poller retries the create from the row if it failed), then evaluate. An earlier head's open check is left to conclude on its + own (fork verdict or timeout): a late-delivered event for an old head must never touch + the current head's check. +- **Conclude** — the verdict is the head's own suite. Once the fork reflects the head (below) + and its dependency jobs settled, every CI test the fork declares is dispatched once, one + run per `ci_test_reference` row the way a deploy of that item would (`trigger_all_ci_tests`, + as the fork's owner, the user who created it, with no more reach than they have; a + missing or disabled owner concludes the check as failure; and without the per-item + debounce so a deploy-triggered run of the same test cannot supersede a suite run), and the job ids are + recorded on the synced-head row (`ci_test_job_ids`; `tests_dispatched_at` claims the + dispatch so the per-job hook and the poller queue it once, and a claim that never recorded + ids is retaken after 5 min). The verdict is exactly those runs: fail-fast on any + failed/canceled; `success` once all settle ("No CI tests" when the fork declares none); + `skipped` ignored. Nothing older, newer or workspace-wide stands in + for a head's runs, so a re-fired event reads the same runs and gets the same answer, a + test-only change is run because the suite runs on every head, and a deploy in flight in + the fork cannot feed another head's check. Runs purged by job retention reset the row so + the head is re-tested. +- **Readiness** — the suite is dispatched only once the fork reflects the head, so it does + not matter which webhook GitHub delivers first. The evidence is `git_sync_synced_head`: the + pull completion hook writes a row when a pull job succeeds (the pull script reports the + commit its clone checked out; the enqueue-time marker is the fallback), and the push + completion hook writes one from the deploy push script's `{pushed, sha, branch, rebased}` + result (a rebased push sits on unpulled commits and is not recorded). The head is ready + when the repository branch's newest row names it, so a branch reset to an older commit + waits for its re-pull; the prune keeps each repository branch's newest row so a PR reopened + at an unchanged head stays ready. This is a sync event log, deliberately apart from + `auto_pull.last_synced_sha`: that map decides whether the next poll pulls (a push must + never write it, or a commit someone else pushed under ours would be skipped) and it is + client-round-tripped settings. The check row stores `head_ref` for the lookup. Dispatch + also waits while a dependency job in the fork or a pull of the repository branch is queued + (a deploy push lands on whatever the remote held when it cloned, so a commit pushed there + from outside is in the workspace only once its pull ran), and the check fails outright if + a dependency job failed after the head's pull started (the item deployed nothing + runnable). A commit the fork never comes to reflect times out; a timeout on a repository + pinned to a sync script older than the one that reports pushed commits names that as the + reason. Needs the hub script versions that report the sha (`LATEST_GIT_SYNC_SCRIPT_PATH`, + `GIT_SYNC_PULL_SCRIPT_PATH`). +- **Drivers** — a per-`ci_test`-job completion hook (low latency) and the git-sync poller + (the backstop: retries the GitHub create/deliver, times stuck checks out after 30 min, + prunes old rows; runs after the auto-pull advisory lock is released so its GitHub calls + never extend the tick). Both call one idempotent `evaluate_and_conclude`, which claims the + decision with a guarded `UPDATE ... WHERE NOT concluded RETURNING` (exactly-once) and + decouples GitHub delivery via `github_posted` so a failed PATCH is retried, not hung. + +Invariants: only a head in the base repo can map to a workspace (a contributor fork's +branch names mean nothing here); the webhook's workspace posts through its own +installation; the timeout stops a hung test job from blocking a required check forever; +rows cascade away with either workspace. A plain feature-branch or +contributor-fork PR resolves to no fork workspace and gets an already-concluded `skipped` +check (branch protection counts `skipped` as passing, so requiring the check does not block +those PRs). A timeout on a repository pinned to a sync script older than the one that reports +pushed commits names that as the reason. Known limit (accepted for v1): the fork's status is workspace-wide (all its +tested items), which for the one-fork-per-PR model equals the PR's scope. + ## 16. Alternatives considered **Portal as webhook proxy (the rejected "option 2").** Subscribe the managed app diff --git a/docs/reusable-ai-agents.md b/docs/reusable-ai-agents.md index 8383bc2836..edb49d75a1 100644 --- a/docs/reusable-ai-agents.md +++ b/docs/reusable-ai-agents.md @@ -16,28 +16,63 @@ every workspace via the standard cached-resource-type sync, like other built-in - The brain config and tools are resolved at runtime from the resource (`windmill-worker/src/ai_executor.rs`): the brain is interpolated, so a nested provider `$res:` credential resolves automatically. -- The step keeps only the flow-local inputs (`user_message`, `user_attachments`) in its own - `input_transforms`; the brain and tools stay in the resource (read-only in the step). +- The step keeps only the flow-local inputs (`user_message`, `user_attachments`, `enabled_tools`) + in its own `input_transforms`; the brain and tools stay in the resource (read-only in the step). + `enabled_tools` says which of the roster this step may call, narrowing one use of a shared agent + without touching the agent: an absent field carries every tool, a list carries the ones it names, + and an empty list carries none. - The agent carries its tools' default input bindings verbatim as authored (static, AI-filled, or flow expressions), so saving round-trips losslessly. Each host flow overrides what it needs: `tool_inputs` stores per-tool overrides (a diff from the resource tool's own transforms) that overlay onto the matching tools at runtime. Editing on a linked step edits - the flow's use of the agent; editing under the "Editing" banner edits the agent itself. + the flow's use of the agent; editing in the agent editor edits the agent itself. In the flow editor, the AI agent step's **Step Input** tab shows a single read-only card (*linked to *, with the inherited brain + tools and an explanatory tooltip) plus -*Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink* -(fork the resolved config — including any `tool_inputs` — back into the step as a one-off). -While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking -first when there is something to drop), and the unsaved-changes badge opens a diff against the -deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of, -and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so -a reload brings the step back as a standalone agent with no path to save back to. +*Edit*, which opens the agent editor over the flow, and *Unlink* (fork the resolved config — +including any `tool_inputs` — back into the step as a one-off). A linked agent's tools appear as display-only graph tool nodes (clicking one selects the agent step); below the step's inputs, each tool gets a section with the standard schema-aware input editors (prop picker included) and a read-only view of its code — edits persist into `tool_inputs`. +## Drafts + +The agent editor edits the resource through a **per-user resource draft** (`draft` table, +`item_kind = 'resource'`), autosaved by `useAgentDraft` and deployed by the editor's own Deploy +button. It is the same draft row the generic resource editor writes and the Review & Deploy page +lists, so an agent can be deployed from any of them. + +A flow does not wait for that deploy to see the draft: + +- Testing the flow, or a single linked step, runs the draft. `runFlowPreview` and `ModuleTest` + substitute each linked step for the standalone step the draft would run as + (`linkedAgentDrafts.ts`): `agent` cleared, the draft's brain as static input transforms, the + draft's tools on the step, and the step's own flow-local inputs kept on top — + the same overlay order `ai_executor.rs` applies to a linked step. `tool_inputs` is untouched, + since the worker overlays it in both branches. +- The step's linked card and the graph's tool nodes show the draft, with a *Draft* badge, so the + editor describes what a test would run. Read-only surfaces (the deployed flow page, the run + viewer) stay on the deployed agent: they resolve tools through `publishLinkedAgentTools` without + the draft flag. +- Deploying the flow lists every linked agent that has a draft in the confirmation dialog, beside + the draft triggers. Deploying one writes the resource and drops the draft; leaving one out keeps + its draft untouched, and the flow runs the agent as currently deployed. That is the one place + the two kinds differ: an undeployed draft trigger is deleted, because it belongs to the flow, + while an agent draft belongs to a resource other flows also use. + +Because a draft is per-user, a flow test can behave differently for two people looking at the same +flow. That is the same contract as a flow draft, and deploying the agent is what makes it shared. + +Inlining has a consequence worth knowing: a preview job's `raw_flow` then carries the agent's +config, where a linked step used to carry only the path and leave the resolution to the worker. So +an agent's prompt and tool set are readable by whoever can read that preview job, which is a wider +set than whoever can read the resource when the agent sits in a more restricted folder than the +flow. No credential travels with it — the provider stays a `$res:` reference, resolved at run time +as the runner. The agent editor's own test pane has inlined the same way since drafts existed; +closing the gap would mean the preview carrying a draft *reference* the worker resolves, rather +than the config. + Sharing works through standard resource folder permissions (save agents under `f/...`). Only the agent's brain is interpolated when the step runs. A tool's own `$res:`/`$var:` defaults are diff --git a/docs/wac-sdk-e2e.md b/docs/wac-sdk-e2e.md index 5c6682d93d..0920bdf168 100644 --- a/docs/wac-sdk-e2e.md +++ b/docs/wac-sdk-e2e.md @@ -34,8 +34,12 @@ cp python-client/wmill/wmill/client.py \ /tmp/windmill-mytest/cache/python_3_12/wmill==*/wmill/client.py find /tmp/windmill-mytest -name __pycache__ -type d -exec rm -rf {} + -# 4. RESTART the backend — see below -# 5. run your scenarios, and rm -rf /tmp/windmill-mytest when done +# 4. drop the bundle snapshots — a bun job runs a bundle built from the package, +# cached by content hash, so a job that already ran keeps the old SDK inlined +rm -rf /tmp/windmill-mytest/cache/bun + +# 5. RESTART the backend — see below +# 6. run your scenarios, and rm -rf /tmp/windmill-mytest when done ``` ## Restart the workers after injecting diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7dde8c270e..c067498797 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.812.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.812.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -5579,9 +5579,9 @@ } }, "node_modules/driver.js": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.3.6.tgz", - "integrity": "sha512-g2nNuu+tWmPpuoyk3ffpT9vKhjPz4NrJzq6mkRDZIwXCrFhrKdDJ9TX5tJOBpvCTBrBYjgRQ17XlcQB15q4gMg==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.8.0.tgz", + "integrity": "sha512-+8/IO7h1v14IzWh2GP60N7T3PFZweXwdn5e5POuxRSBoCYUojsBxzqawPeXh3YZIibRy7EehYNEyxe7slwwtdg==", "license": "MIT" }, "node_modules/dts-bundle-generator": { diff --git a/frontend/package.json b/frontend/package.json index 593ad0e404..fee148af8a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.812.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/actingUser.svelte.ts b/frontend/src/lib/actingUser.svelte.ts new file mode 100644 index 0000000000..30da65ff83 --- /dev/null +++ b/frontend/src/lib/actingUser.svelte.ts @@ -0,0 +1,74 @@ +import { untrack } from 'svelte' +import { fromStore } from 'svelte/store' +import { SvelteMap } from 'svelte/reactivity' +import { userStore, workspaceStore, type UserExt } from '$lib/stores' +import { getWorkspaceRole, type RoleLookup } from '$lib/user' + +/** + * The user acting in a workspace that is not necessarily the one the top nav points at — an AI + * session or a workspace-specific variant acts on a workspace the nav deliberately is not on. + * + * `$userStore` answers for the navigation workspace at no cost, exactly as every permission check + * in the app did before this hook existed — including when it holds nobody, which reads as unknown + * and refuses. Every other workspace is looked up, and an unresolved user there is `undefined`: it + * must never fall back to the navigation user, whose rights belong to another workspace. + * `canWrite`/`isOwner` refuse for an unknown user, which is the only safe answer. A caller that + * must not render that refusal as a denial asks `resolved` first. + */ +export function useActingUser(workspace: () => string | undefined) { + const navWorkspace = fromStore(workspaceStore) + const navUser = fromStore(userStore) + const looked = new SvelteMap() + // The workspace this effect last acted on, so arriving at one is distinguishable from the + // effect re-running while already there. + let asking: string | undefined + + $effect(() => { + const ws = workspace() + if (asking !== ws) { + asking = ws + // Dropped on the way *in*, not on the way out: a lookup that fails after the acting + // workspace has already moved on has no entry to clear at the moment it is left, so + // clearing it there would keep a refusal that no attempt is behind any more. + if (ws && untrack(() => looked.get(ws)?.kind) === 'lookup_failed') looked.delete(ws) + } + if (!ws || ws === navWorkspace.current) return + // Any settled answer stops the asking, a failure included — otherwise recording one + // would re-enter this effect and loop. + if (looked.has(ws)) return + untrack(() => { + // Memoized process-wide, so two components pointed at the same workspace share one + // request rather than each issuing their own. + getWorkspaceRole(ws).then((lookup) => looked.set(ws, lookup)) + }) + }) + + function userIn(ws: string | undefined): UserExt | undefined { + if (!ws) return undefined + if (ws === navWorkspace.current) return navUser.current + const lookup = looked.get(ws) + return lookup?.kind === 'resolved' ? lookup.user : undefined + } + + return { + /** The acting user in `ws`, or `undefined` when it is not known. Only workspaces this + * hook has been pointed at are looked up; the rest read as unknown. */ + in: userIn, + /** Whether `ws` has an answer at all — a user, or a lookup that came back without one. + * The navigation workspace always has one: `$userStore`, "nobody" included. */ + resolved: (ws: string | undefined): boolean => + !!ws && (ws === navWorkspace.current || looked.has(ws)), + get current(): UserExt | undefined { + return userIn(workspace()) + }, + /** Drop the lookups that came back empty so they are asked again. Arriving at a + * workspace already does this; a long-lived editor must call this too when it starts a + * fresh session on the workspace it is already on, or a `whoami` that happened to fail + * pins it to "unknown user" for as long as it stays there. */ + forgetFailures(): void { + for (const [ws, lookup] of looked) { + if (lookup.kind === 'lookup_failed') looked.delete(ws) + } + } + } +} diff --git a/frontend/src/lib/attachments/newTabModifier.dom.test.ts b/frontend/src/lib/attachments/newTabModifier.dom.test.ts new file mode 100644 index 0000000000..115877298c --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.dom.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { newTabModifier } from './newTabModifier.svelte' + +const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent }) +const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)' +const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + +const attached: (() => void)[] = [] + +/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */ +function pill() { + const node = document.createElement('span') + document.body.append(node) + const modifier = newTabModifier() + const cleanup = modifier.attach(node) as () => void + attached.push(cleanup) + const hover = (init: MouseEventInit = {}) => + node.dispatchEvent(new MouseEvent('mouseenter', init)) + const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init)) + const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave')) + return { modifier, hover, move, unhover, cleanup } +} + +const keydown = (init: KeyboardEventInit) => + window.dispatchEvent(new KeyboardEvent('keydown', init)) + +describe('newTabModifier', () => { + // The window listeners outlive the DOM, so every case has to be torn down through the + // attachment rather than by emptying the body. + afterEach(() => { + attached.splice(0).forEach((cleanup) => cleanup()) + document.body.replaceChildren() + vi.unstubAllGlobals() + }) + + // The hover event carries the live modifier state, so a modifier pressed before the pointer + // arrived (or while this window was unfocused) is picked up rather than read as false. + it('seeds from the hover event, per platform', () => { + onPlatform(LINUX) + const linux = pill() + linux.hover({ ctrlKey: true }) + expect(linux.modifier.held).toBe(true) + + onPlatform(MAC) + const mac = pill() + // macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier. + mac.hover({ ctrlKey: true }) + expect(mac.modifier.held).toBe(false) + mac.hover({ metaKey: true }) + expect(mac.modifier.held).toBe(true) + }) + + // Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase + // listener would go blind whenever focus sits in one. + it('sees a keydown that a focused element stops from propagating', () => { + onPlatform(LINUX) + const { modifier, hover } = pill() + hover() + const input = document.createElement('input') + input.addEventListener('keydown', (e) => e.stopPropagation()) + document.body.append(input) + + input.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true }) + ) + expect(modifier.held).toBe(true) + }) + + // A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown + // on the way back, while the pointer parked on the pill fires no fresh mouseenter either. + it('re-seeds from pointer movement after the window lost focus', () => { + onPlatform(LINUX) + const { modifier, hover, move } = pill() + hover({ ctrlKey: true }) + window.dispatchEvent(new Event('blur')) + expect(modifier.held).toBe(false) + + move({ ctrlKey: true }) + expect(modifier.held).toBe(true) + }) + + it('stops tracking once unhovered', () => { + onPlatform(LINUX) + const { modifier, hover, unhover } = pill() + hover({ ctrlKey: true }) + unhover() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) + + it('stops tracking when the element is destroyed while hovered', () => { + onPlatform(LINUX) + const { modifier, hover, cleanup } = pill() + hover({ ctrlKey: true }) + // Hovering again without leaving must not strand the first hover's listeners, which nothing + // would then hold a reference to. + hover({ ctrlKey: true }) + cleanup() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) +}) diff --git a/frontend/src/lib/attachments/newTabModifier.svelte.ts b/frontend/src/lib/attachments/newTabModifier.svelte.ts new file mode 100644 index 0000000000..06c69f36cc --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.svelte.ts @@ -0,0 +1,66 @@ +import type { Attachment } from 'svelte/attachments' +import { isMac } from '$lib/utils' + +/** + * Tracks whether the modifier that turns a click into a new browser tab is held, but only while + * the attached element is hovered, which is the only moment the answer is used. + */ +export function newTabModifier() { + let held = $state(false) + + // Only the modifier that actually yields a tab: shift opens a window, alt can start a + // download, and on macOS ctrl+click is a secondary click. + // Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to + // a focus change cannot strand the flag on. + const sync = (event: KeyboardEvent | MouseEvent) => { + held = isMac() ? event.metaKey : event.ctrlKey + } + const clear = () => { + held = false + } + + const attach: Attachment = (node) => { + // One controller per hover: a mirrored remove list leaks any listener whose options drift. + let hover: AbortController | undefined + const leave = () => { + hover?.abort() + hover = undefined + clear() + } + const enter = (event: MouseEvent) => { + // Seeded from the hover itself: mouse events carry the same modifier flags as key events, + // so a modifier already held before the pointer arrived reads correctly. + sync(event) + // Re-entering without an intervening leave would strand the previous controller: nothing + // else references it, so its listeners could never be removed. + hover?.abort() + hover = new AbortController() + const { signal } = hover + // Same reason the hover seeds: a modifier held across a keyboard app switch delivers no + // keydown on the way back, so the pointer is all that is left to re-read it from. + node.addEventListener('mousemove', sync, { signal }) + // Capture: editors and menus stopPropagation the keys they handle, hiding the modifier + // from a bubble-phase listener whenever focus sits in one. + window.addEventListener('keydown', sync, { capture: true, signal }) + window.addEventListener('keyup', sync, { capture: true, signal }) + // Not capture, unlike the two above: blur does not bubble but does reach the window while + // capturing, so it would fire for every element that loses focus. + window.addEventListener('blur', clear, { signal }) + } + + const life = new AbortController() + node.addEventListener('mouseenter', enter, { signal: life.signal }) + node.addEventListener('mouseleave', leave, { signal: life.signal }) + return () => { + life.abort() + leave() + } + } + + return { + get held() { + return held + }, + attach + } +} diff --git a/frontend/src/lib/cloud.ts b/frontend/src/lib/cloud.ts index dbf4a58f48..30b2e54b75 100644 --- a/frontend/src/lib/cloud.ts +++ b/frontend/src/lib/cloud.ts @@ -6,6 +6,15 @@ export function isCloudHosted(): boolean { // may be missing or a stub with no `location`. Same defensive shape as // `isChromiumBrowser`. if (!BROWSER) return false + // Dev only: the cloud-specific UI (quotas, plan upgrade, the pre-approved trial offer) + // is otherwise unreachable from localhost. `localStorage.cloudHostedOverride = '1'` opts + // a browser in against a backend started with CLOUD_HOSTED. + if ( + import.meta.env.DEV && + globalThis.window?.localStorage?.getItem('cloudHostedOverride') === '1' + ) { + return true + } return globalThis.window?.location?.hostname == 'app.windmill.dev' } diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index e4ee3ca7b9..dedeee90e2 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -12,6 +12,7 @@ import { Loader2 } from 'lucide-svelte' import { untrack } from 'svelte' import GitHubAppIntegration from './GitHubAppIntegration.svelte' + import GitLabIntegration from './GitLabIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' import ResourceGen from './copilot/ResourceGen.svelte' @@ -28,6 +29,13 @@ isValid?: boolean linkedSecretCandidates?: string[] | undefined description?: string | undefined + /** Workspace the resource is being saved into, which is not always the one + * being navigated. The GitLab picker has to store the credential where the + * resource will look for it. */ + workspace?: string + /** Fired once the GitLab picker has stored the picked project's token, so a + * form that would otherwise file the URL as a secret knows it holds none. */ + onCredentialStored?: () => void onSynced?: () => void } @@ -39,6 +47,8 @@ isValid = $bindable(true), linkedSecretCandidates = undefined, description = $bindable(undefined), + workspace = undefined, + onCredentialStored, onSynced = undefined }: Props = $props() @@ -249,6 +259,19 @@ }} onDescriptionUpdate={(newDescription) => (description = newDescription)} /> + + { + args = newArgs + rawCode = JSON.stringify(args, null, 2) + rawCodeEditor?.setCode(rawCode) + }} + /> {#if resourceType?.includes('bedrock') && !isCloudHosted()} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 320b3e36dd..cef58fcdb2 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -5,10 +5,12 @@ import LabelsInput from './LabelsInput.svelte' import IconedResourceType from './IconedResourceType.svelte' import { + addResourceTypeDisplayName, isCustomResourceTypeName, resourceTypeDisplayName, resourceTypeMatchRank, resourceTypeSearchText, + setResourceTypeDisplayNames, sortResourceTypesByMatch } from './resourceTypeDisplay' import { @@ -41,8 +43,16 @@ import TextInput from './text_input/TextInput.svelte' import { sameTopDomainOrigin } from '$lib/cookies' import SyncResourceTypes from './SyncResourceTypes.svelte' + import { + alphabetical, + byPopularity, + hubResourceTypePicks, + localResourceTypeCounts, + recordHubResourceTypePick + } from './pickerPopularity' import Label from './Label.svelte' import ResourcePathHint from './ResourcePathHint.svelte' + import SchemaForm from './SchemaForm.svelte' interface Props { step?: number @@ -223,6 +233,28 @@ | undefined ) + /** Fields of the resource type the provider's registry entry asks for once the token is + * in (`resource_fields`): what no token response carries, like Snowflake's database. A + * list rather than "every other field" because most OAuth types also hold the fields of + * another way in: ServiceNow's basic-auth password, Bitbucket's app password. */ + let resourceFields = $derived((registryEntry()?.resource_fields as string[] | undefined) ?? []) + + /** Their slice of the resource type's schema, so they render with the type's own + * descriptions; plain text inputs while the type is not synced from the hub. */ + let resourceFieldsSchema = $derived.by(() => { + const props: Record = + (resourceTypeInfo?.schema as any)?.properties ?? {} + return { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + order: resourceFields, + properties: Object.fromEntries( + resourceFields.map((f) => [f, props[f] ?? { type: 'string', description: '' }]) + ), + required: [] + } + }) + /** Instance entry declares client credentials but not authorization_code * (custom provider configured with only a token URL) */ let authCodeUnavailable = $state(false) @@ -332,6 +364,7 @@ export async function open(rt?: string) { if (!rt) { loadResourceTypes() + loadPopularity() } step = 1 //express && !manual ? 3 : 1 // The list is keyboard-driven from the search field, so it takes focus on open. @@ -378,12 +411,29 @@ } } + /** + * Orders the browse list: the types this workspace already has resources of lead, ranked + * among themselves by the hub's pick counts, then everything else on the same counts. + * `byPopularity` carries the full rule. Both signals are fetched, so the rows render + * alphabetically and re-sort when this lands. + */ + let popularity: (a: string, b: string) => number = $state(alphabetical) + + async function loadPopularity() { + if (!effectiveWorkspace) return + const [hub, local] = await Promise.all([ + hubResourceTypePicks(effectiveWorkspace), + localResourceTypeCounts(effectiveWorkspace) + ]) + popularity = byPopularity(hub, local) + } + async function loadConnects() { if (!connects) { try { - const list = (await OauthService.listOauthConnects()) - .filter((x) => x.name != 'supabase_wizard') - .sort((a, b) => a.name.localeCompare(b.name)) + const list = (await OauthService.listOauthConnects()).filter( + (x) => x.name != 'supabase_wizard' + ) connects = list.map((x) => x.name) connectsInfo = Object.fromEntries(list.map((x) => [x.name, x])) } catch (e) { @@ -449,6 +499,7 @@ // $derived, so search re-ranks when they land. ResourceService.listResourceType({ workspace: effectiveWorkspace }) .then((types) => { + setResourceTypeDisplayNames(types) resourceTypeDescriptions = Object.fromEntries( types.filter((t) => t.description).map((t) => [t.name, t.description!]) ) @@ -466,19 +517,17 @@ // providers — so any of them can also be connected with the user's own // credentials or manually, not only via the shared instance setup (same as // the authorization-code behavior). - connectsManual = availableRts - .map( - (x) => - ({ - key: x, - ...(apiTokenApps[x] ?? { - instructions: '', - img: undefined, - linkedSecret: undefined - }) - }) as { key: string; img?: string; instructions: string[] } - ) - .sort((a, b) => a.key.localeCompare(b.key)) + connectsManual = availableRts.map( + (x) => + ({ + key: x, + ...(apiTokenApps[x] ?? { + instructions: '', + img: undefined, + linkedSecret: undefined + }) + }) as { key: string; img?: string; instructions: string[] } + ) const filteredNativeLanguages = filteredConnectsManual?.filter( (o) => nativeLanguagesCategory?.includes(o[0]) ?? false ) @@ -606,6 +655,7 @@ workspace: effectiveWorkspace, path: resourceType }) + addResourceTypeDisplayName(resourceTypeInfo) const props: Record = resourceTypeInfo?.schema?.['properties'] ?? {} const newArgsKeys = Object.keys(props).filter((x) => props?.[x]?.type == 'string') ?? [] @@ -623,9 +673,11 @@ export async function next() { if (step == 1) { linkedSecrets = [] + // Both branches: the OAuth one fills `resourceFields` into the same map, and fields + // typed into another type's form before Back would otherwise ride along. + args = {} if (manual) { getResourceTypeInfo() - args = {} } else { getResourceTypeInfo() // Awaited: the popup is built from `scopes`, so advancing before this @@ -864,10 +916,19 @@ ) } - const resourceValue = args + // A copy: the form is still mounted and bound to `args` across the awaits below, and + // puts back the default of any field removed from it. + const resourceValue = $state.snapshot(args) let savedVariableCount = 0 if (!manual) { + // A field left blank is absent, not an empty string a consumer reads as a value: + // the Snowflake executor sends any `database` it finds, empty or not. + for (const f of resourceFields) { + if (resourceValue[f] === '' || resourceValue[f] == undefined) { + delete resourceValue[f] + } + } // OAuth flow: single secret variable for the token if (typeof value == 'string' && value != '' && !value.startsWith('$var:')) { savedVariableCount++ @@ -952,6 +1013,10 @@ } }) } + // Saving is what "picking a type" means to the hub: reaching step 2 is still + // browsing. Both branches above count, `filling` included — an imported stub is + // a type taken into the workspace just the same. + recordHubResourceTypePick(effectiveWorkspace, resourceType) dispatch('refresh', path) dispatch('close') sendUserToast( @@ -972,6 +1037,9 @@ if (step == 1) { loadConnects() loadResourceTypes() + // Opened on a specific type, `open()` skipped this; backing out to the browse + // list is the first time it is needed. + loadPopularity() } } @@ -980,24 +1048,35 @@ let filteredConnects: { key: string }[] = $state([]) let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([]) - // uFuzzy scores the name and the description as one string, so searching "google" ranks - // every type whose description mentions Google alongside the ones named after it. Re-sort - // on which field matched, keeping uFuzzy's order within a tier. + let searching = $derived(filter.trim() !== '') + + // Searching, the query owns the order: uFuzzy scores the name and the description as one + // string, so "google" ranks every type whose description mentions Google alongside the + // ones named after it — re-sort on which field matched, keeping uFuzzy's order within a + // tier. Browsing, there is no query to rank against, so popularity orders the list. const rank = (items: { key: string }[] | undefined) => items && - sortResourceTypesByMatch( - items, - filter, - (x) => x.key, - (x) => resourceTypeDescriptions[x.key] - ) + (searching + ? sortResourceTypesByMatch( + items, + filter, + (x) => x.key, + (x) => resourceTypeDescriptions[x.key] + ) + : // Both signals are keyed by resource type, and a sandbox client is a second row + // against one (`salesforce_sandbox` saves a `salesforce`), so it ranks on the + // parent's popularity. Its own key still breaks the tie the pair then have, or + // the two would order arbitrarily. + [...items].sort( + (a, b) => + popularity(stripSandboxSuffix(a.key), stripSandboxSuffix(b.key)) || + a.key.localeCompare(b.key) + )) let rankedConnects = $derived(rank(filteredConnects)) let rankedConnectsManual = $derived( rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined ) - let searching = $derived(filter.trim() !== '') - // Browsing, the "Others" list leads with the native database types. Searching, that // grouping would outrank the search itself — `ms_sql_server` sorting under `mysql` on // "sql" — so the ranked order stands on its own. @@ -1332,6 +1411,15 @@ {linkedSecretCandidates} {resourceType} {resourceTypeInfo} + workspace={effectiveWorkspace} + onCredentialStored={() => { + // `forceSecretValue` files a git_repository's `url` in a secret + // variable, for the URLs that carry a token in them. The picker's + // does not: the token is stored separately, so that variable would + // hold nothing secret and add a second place to keep in step with + // the resource. + linkedSecrets = linkedSecrets.filter((f) => f !== 'url') + }} bind:args bind:isValid onSynced={getResourceTypeInfo} @@ -1512,6 +1600,17 @@ {/if} + + {#if step == 4 && !manual && !express && !fillPath && resourceFields.length > 0} + + {/if} {#if apiTokenApps[resourceType] || !manual}
  • diff --git a/frontend/src/lib/components/AppTutorials.svelte b/frontend/src/lib/components/AppTutorials.svelte deleted file mode 100644 index 25d06e0167..0000000000 --- a/frontend/src/lib/components/AppTutorials.svelte +++ /dev/null @@ -1,29 +0,0 @@ - - - diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 7f50794a52..b7001059c2 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -1078,6 +1078,7 @@ {otherArgs} {helperScript} {workspace} + {disabled} bind:value format={format ?? ''} /> diff --git a/frontend/src/lib/components/CenteredModal.svelte b/frontend/src/lib/components/CenteredModal.svelte index 3250b3a16a..08234c25b1 100644 --- a/frontend/src/lib/components/CenteredModal.svelte +++ b/frontend/src/lib/components/CenteredModal.svelte @@ -14,6 +14,8 @@ centerVertically?: boolean loading?: boolean containOverflow?: boolean + /** The Windmill version and update notice in the header. */ + showVersion?: boolean children?: import('svelte').Snippet } @@ -25,6 +27,7 @@ centerVertically = true, loading = false, containOverflow = false, + showVersion = true, children }: Props = $props() @@ -82,5 +85,5 @@ {/if} - + diff --git a/frontend/src/lib/components/ChangeInstanceUsername.svelte b/frontend/src/lib/components/ChangeInstanceUsername.svelte index 1fcd726cea..fce0f4e8f9 100644 --- a/frontend/src/lib/components/ChangeInstanceUsername.svelte +++ b/frontend/src/lib/components/ChangeInstanceUsername.svelte @@ -3,6 +3,7 @@ import Popover from './meltComponents/Popover.svelte' import { autoPlacement } from '@floating-ui/core' import ChangeInstanceUsernameInner from './ChangeInstanceUsernameInner.svelte' + import { AlertTriangle } from 'lucide-svelte' interface Props { email: string @@ -24,9 +25,22 @@ closeButton > {#snippet trigger()} - + {#if isConflict} + + + {/if} {/snippet} {#snippet content()} | undefined - dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps + dbTableOpsFactory: (params: { + colDefs: ColumnDef[] + tableKey: string + /** Raw SQL predicate AND-ed into the reads (already escaped). */ + whereClause?: string + }) => IDbTableOps dbSchemaOps: IDbSchemaOps refresh?: () => void initialSchemaKey?: string @@ -214,6 +219,83 @@ : selected.tableKey ) + // Set by "Go to row" on a foreign-keyed cell; pinned to the table it was + // created for so a schema change can't carry it onto an unrelated table. + let rowFilter: (DbRowFilter & { tableKey: string }) | undefined = $state() + let activeRowFilter = $derived(rowFilter?.tableKey === tableKey ? rowFilter : undefined) + let whereClause = $derived( + activeRowFilter + ? renderDbEqualityFilter(activeRowFilter.column, activeRowFilter.value, dbType) + : undefined + ) + + function selectTable(schemaKey: string | undefined, table: string) { + rowFilter = undefined + selected = { schemaKey, tableKey: table } + } + + /** Where a foreign key's `schema.table` target lives in the sidebar, or + * undefined when it cannot be opened from here. */ + function resolveForeignKeyTarget( + targetTable: string + ): { schemaKey: string; table: string } | undefined { + const parts = targetTable.split('.') + const table = parts[parts.length - 1] + const qualifier = parts.length > 1 ? parts.slice(0, -1).join('.') : undefined + // Without schema support the sidebar browses the connection's default + // schema only, and unqualified reads would hit a same-named local table. + if (!dbSupportsSchemas && qualifier && qualifier !== selected.schemaKey) return undefined + const schemaKey = dbSupportsSchemas && qualifier ? qualifier : selected.schemaKey + if (!schemaKey || !(table in (dbSchema.schema[schemaKey] ?? {}))) return undefined + return { schemaKey, table } + } + + function goToRow(target: DbForeignKeyTarget) { + const resolved = resolveForeignKeyTarget(target.table) + if (!resolved) { + sendUserToast(`Table ${target.table} cannot be opened from this schema`, true) + return + } + if (renderDbEqualityFilter(target.column, target.value, dbType) === undefined) { + sendUserToast('This value cannot be used as a filter', true) + return + } + const { schemaKey, table } = resolved + selectTable(schemaKey, table) + rowFilter = { + tableKey: dbSupportsSchemas ? `${schemaKey}.${table}` : table, + column: target.column, + value: target.value + } + } + + // The result carries the table it was fetched for: `resource` keeps the + // previous value while refetching, and a stale list would decorate the new + // table's same-named columns as foreign keys. + let foreignKeys = resource( + [() => selected.tableKey, () => selected.schemaKey, () => colDefs], + async ([table, schema], _prev, { signal }) => { + if (!table) return undefined + const forTableKey = dbSupportsSchemas && schema ? `${schema}.${table}` : table + const fks = + features?.foreignKeys === false + ? [] + : await dbSchemaOps.onFetchForeignKeys({ table, schema }) + // A newer selection started meanwhile: an AbortError keeps this result + // out of `current`, where it would shadow the newer table's keys. + if (signal.aborted) throw new DOMException('Superseded', 'AbortError') + return { tableKey: forTableKey, foreignKeys: fks } + } + ) + // Only keys whose target the sidebar can open get the "Go to row" affordance. + let currentForeignKeys = $derived.by(() => { + const fetched = foreignKeys.current + if (!fetched || fetched.tableKey !== tableKey) return undefined + return fetched.foreignKeys.filter( + (fk) => fk.targetTable && resolveForeignKeyTarget(fk.targetTable) !== undefined + ) + }) + let askingForConfirmation: | (ConfirmationModal['$$prop_def'] & { onConfirm: () => void }) | undefined = $state() @@ -395,14 +477,12 @@ role="button" tabindex="0" onclick={() => { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) }} onkeydown={(e) => { if (e.key === 'Enter' || e.key === ' ') { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) } }} @@ -468,7 +548,7 @@ + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte index ae5182f560..3bda6aa216 100644 --- a/frontend/src/lib/components/DdlMigrationGuard.svelte +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -3,7 +3,7 @@ import Modal2 from './common/modal/Modal2.svelte' import NewDataTableMigrationModal from './workspaceSettings/NewDataTableMigrationModal.svelte' import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' - import { splitSqlStatements, isDdlStatement } from './sqlDdl' + import { joinSqlStatements, splitSqlRuns } from './sqlDdl' import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry' import { CornerDownLeft } from 'lucide-svelte' @@ -11,7 +11,7 @@ type Choice = 'run' | 'migrate' | 'cancel' - let promptStatement = $state(undefined) + let promptStatements = $state([]) let promptOpen = $state(false) let resolvePrompt: ((choice: Choice) => void) | undefined = undefined let resolveMigrationClosed: ((created: boolean) => void) | undefined = undefined @@ -23,11 +23,15 @@ // toast action after a migration is created here. let migrationsModal = $state(undefined) + // The block is shown as-is in the prompt and becomes the migration body, where + // every statement inside the BEGIN; ... END; frame must be `;`-terminated. + let promptSql = $derived(joinSqlStatements(promptStatements)) + function finishPrompt(choice: Choice) { const r = resolvePrompt resolvePrompt = undefined promptOpen = false - promptStatement = undefined + promptStatements = [] r?.(choice) } @@ -49,10 +53,10 @@ } } - function promptDdl(statement: string): Promise { + function promptDdl(statements: string[]): Promise { return new Promise((resolve) => { resolvePrompt = resolve - promptStatement = statement + promptStatements = statements promptOpen = true }) } @@ -66,49 +70,51 @@ // Open the prefilled new-migration modal. Resolves with whether a migration // was actually created (false if the user cancelled / closed it). - function openMigrationModal(statement: string): Promise { + function openMigrationModal(sql: string): Promise { return new Promise((resolve) => { resolveMigrationClosed = (created: boolean) => resolve(created) - newMigrationModal?.open({ codeUp: statement }) + newMigrationModal?.open({ codeUp: sql }) }) } /** - * Inspect `code` for DDL statements. For each one, prompt the user to run it - * anyway or turn it into a migration (prompts shown one at a time). Returns - * whether to proceed and the code to run (with migrated statements stripped). + * Inspect `code` for DDL statements. Each run of adjacent DDL statements is + * prompted for once (runs shown one at a time) and becomes a single migration, + * so a chain of schema changes applies in one transaction instead of asking + * once per statement. Returns whether to proceed and the code to run (with + * migrated statements stripped). */ export async function guard( code: string ): Promise<{ proceed: boolean; code: string; ranMigration: boolean }> { migrationRan = false - const statements = splitSqlStatements(code) - if (!statements.some((s) => isDdlStatement(s))) { + const runs = splitSqlRuns(code) + if (!runs.some((r) => r.isDdl)) { return { proceed: true, code, ranMigration: false } } const kept: string[] = [] - for (const statement of statements) { - if (!isDdlStatement(statement)) { - kept.push(statement) + for (const run of runs) { + if (!run.isDdl) { + kept.push(...run.statements) continue } - // Re-prompt for this statement until the user makes a terminal choice; + // Re-prompt for this run until the user makes a terminal choice; // cancelling the migration modal returns to the prompt with the DDL intact. for (;;) { - const choice = await promptDdl(statement) + const choice = await promptDdl(run.statements) if (choice === 'cancel') { logDdlGuardChoice('cancelled') return { proceed: false, code, ranMigration: migrationRan } } if (choice === 'run') { logDdlGuardChoice('run_anyway') - kept.push(statement) + kept.push(...run.statements) break } - // migrate: only strip the statement once a migration is actually + // migrate: only strip the statements once a migration is actually // created; if the modal was cancelled, loop back to the prompt. - const created = await openMigrationModal(statement) + const created = await openMigrationModal(joinSqlStatements(run.statements)) if (created) { logDdlGuardChoice('migrated') break @@ -116,14 +122,14 @@ } } - return { proceed: true, code: kept.join(';\n'), ranMigration: migrationRan } + return { proceed: true, code: joinSqlStatements(kept), ranMigration: migrationRan } } 1 ? 'Schema changes detected' : 'Schema change detected'} fixedWidth="md" fixedHeight="adaptive" bind:isOpen={promptOpen} @@ -131,12 +137,17 @@ >

    - This looks like a schema-changing (DDL) statement. Schema changes are best tracked as - migrations rather than run ad-hoc. Create a migration for it instead? + {#if promptStatements.length > 1} + These {promptStatements.length} consecutive statements are schema-changing (DDL). Schema changes + are best tracked as migrations rather than run ad-hoc. Create a single migration for them instead? + {:else} + This looks like a schema-changing (DDL) statement. Schema changes are best tracked as + migrations rather than run ad-hoc. Create a migration for it instead? + {/if} 

    {promptStatement ?? ''}
    {promptSql}
    diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index c4674d7313..1bf9e7fde3 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -1245,7 +1245,6 @@ {:else if inputType === 'dynselect'}
    + + Service name, environment, resource attributes, metrics temporality and other + options are set with environment variables. + {/if}
    {:else if setting.fieldType == 'otel_tracing_proxy'} @@ -861,6 +870,8 @@ {:else if setting.fieldType == 'ws_connectivity'} + {:else if setting.fieldType == 'instance_banner'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index d124569929..f38c0d3892 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -14,6 +14,7 @@ import { sleep } from '$lib/utils' import { enterpriseLicense } from '$lib/stores' + import { isCloudHosted } from '$lib/cloud' import { createEventDispatcher } from 'svelte' import { setLicense } from '$lib/enterpriseUtils' @@ -100,7 +101,8 @@ otel: {}, indexer_settings: {}, critical_error_channels: [], - github_enterprise_app: {} + github_enterprise_app: {}, + instance_banner: {} } function applyFormDefaults(vals: Record): void { @@ -132,11 +134,15 @@ } applyFormDefaults(nvalues) - // Apply select/select_python defaults so initialValues matches what InstanceSetting's $effect does + // Apply declared defaults before snapshotting initialValues, so a default shows without + // marking the form dirty: a select's mirrors InstanceSetting's $effect, a boolean's is + // what its toggle shows while the key is unset. for (const category of settingsKeys) { for (const s of settings[category]) { if ( - (s.fieldType === 'select' || s.fieldType === 'select_python') && + (s.fieldType === 'select' || + s.fieldType === 'select_python' || + s.fieldType === 'boolean') && nvalues[s.key] == undefined && s.defaultValue ) { @@ -524,6 +530,10 @@ for (const category of settingsKeys) { const categorySettings = getSettingsForCategory(category) result[category] = categorySettings.some((s) => { + // A field the build never renders must not be able to block Save: off-cloud its + // value is unreachable, so an invalid one (from config sync, say) would leave the + // category permanently unsaveable with nothing on screen to fix. + if (s.cloudonly && !isCloudHosted()) return false if (s.isValid && !s.isValid(currentValues?.[s.key])) return true if (s.validate) { const errors = s.validate(currentValues?.[s.key]) @@ -1061,8 +1071,7 @@
  • worker usage (worker, worker instance, vCPUs, memory)
  • user usage (author count, operator count, the distinct guests of the last 30 days, - the seats they add past the free allowance, and the workspaces that allow - guests)
  • superadmin email addresses
  • development instance status
    • @@ -1075,11 +1084,16 @@ >feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions are started for, whether AI chat skills are turned on or off and how often one is - loaded, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a - membership, the plan tier and quota shown when the execution meter is opened, whether - app sandbox isolation is turned on, whether a step's workspace script is edited from - the flow editor, and how data tables and their migrations are set up and used, last 30 - days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1129,19 +1143,23 @@
  • worker usage (worker, worker instance, vCPUs, memory)
  • user usage (author count, operator count, the distinct guests of the last 30 days, - the seats they add past the free allowance, and the workspaces that allow - guests)
  • development instance status
  • feature usage (counts of which product features are used, including AI provider and model identifiers, the names of public hub scripts used, the languages debug sessions are started for, whether AI chat skills are turned on or off and how often one is - loaded, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a - membership, the plan tier and quota shown when the execution meter is opened, whether - app sandbox isolation is turned on, whether a step's workspace script is edited from - the flow editor, and how data tables and their migrations are set up and used, last 30 - days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index eb6cc9bc97..64ca912608 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -21,9 +21,18 @@ args: any argLabel?: string | undefined workspace?: string | undefined + /** Drop the header's expand-into-a-drawer button, for a caller that already offers a + * way to open the run in full. */ + disableExpand?: boolean } - let { id = undefined, args, argLabel = undefined, workspace = undefined }: Props = $props() + let { + id = undefined, + args, + argLabel = undefined, + workspace = undefined, + disableExpand = false + }: Props = $props() // Internal flag injected by "test this step" runs to suppress the asset // dispatcher. Not a real input: shown as a badge instead of a table row, @@ -125,18 +134,20 @@ ${Object.entries(displayArgs) Value {#snippet headerAction()} -
    - -
    + {#if !disableExpand} +
    + +
    + {/if} {/snippet} diff --git a/frontend/src/lib/components/LightweightResourcePicker.svelte b/frontend/src/lib/components/LightweightResourcePicker.svelte index bb0c21b4c1..58557e8c8b 100644 --- a/frontend/src/lib/components/LightweightResourcePicker.svelte +++ b/frontend/src/lib/components/LightweightResourcePicker.svelte @@ -10,6 +10,7 @@ import Select from './select/Select.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' + import { loadResourceTypeDisplayName } from './displayNameLoaders' interface Props { value: string | undefined @@ -218,6 +219,7 @@ on:click={() => { refreshCount += 1 open = true + if (ws && resourceType) void loadResourceTypeDisplayName(ws, resourceType) drawer?.openDrawer?.() }} startIcon={{ icon: Plus }} diff --git a/frontend/src/lib/components/LocalDraftBanner.svelte b/frontend/src/lib/components/LocalDraftBanner.svelte index dff131b281..af2787222e 100644 --- a/frontend/src/lib/components/LocalDraftBanner.svelte +++ b/frontend/src/lib/components/LocalDraftBanner.svelte @@ -2,12 +2,8 @@ import { Button } from '$lib/components/common' import DiffDrawer from '$lib/components/DiffDrawer.svelte' import { classes } from '$lib/components/common/alert/model' - import { - cleanValueProperties, - orderedYamlStringify, - replaceFalseWithUndefined, - type Value - } from '$lib/utils' + import { cleanValueProperties, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AlertCircle, Diff } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' import { fade } from 'svelte/transition' diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 0307643a08..da032717f6 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -1,4 +1,5 @@
    @@ -31,9 +32,11 @@
    -
    - - -
    + {#if showVersion} +
    + + +
    + {/if}
    diff --git a/frontend/src/lib/components/ModulePreview.svelte b/frontend/src/lib/components/ModulePreview.svelte index a9b769a4a5..be62557b81 100644 --- a/frontend/src/lib/components/ModulePreview.svelte +++ b/frontend/src/lib/components/ModulePreview.svelte @@ -21,6 +21,10 @@ class?: string onJobDone?: () => void hideRunButton?: boolean + /** Passed through to the form: the step whose agent form this preview accompanies. */ + openFieldsKey?: string + /** Passed through to the form: fields it must offer whatever the step holds. */ + runInputKeys?: readonly string[] } let { @@ -34,7 +38,9 @@ focusArg = undefined, class: className = '', onJobDone, - hideRunButton = false + hideRunButton = false, + openFieldsKey = undefined, + runInputKeys = undefined }: Props = $props() const { flowStore } = getContext('FlowEditorContext') @@ -85,5 +91,5 @@ {/if} - + diff --git a/frontend/src/lib/components/ModulePreviewForm.svelte b/frontend/src/lib/components/ModulePreviewForm.svelte index 72bb9fe36b..092eb75706 100644 --- a/frontend/src/lib/components/ModulePreviewForm.svelte +++ b/frontend/src/lib/components/ModulePreviewForm.svelte @@ -15,6 +15,7 @@ import { twMerge } from 'tailwind-merge' import { workspaceStore } from '$lib/stores' import { AGENT_FIELDS, initialVisibleAgentFields } from './flows/agentFormFields' + import { openAgentFields } from './flows/content/AiAgentStepInputs.svelte' interface Props { schema: Schema | { properties?: Record; required?: string[] } @@ -23,6 +24,12 @@ isValid?: boolean autofocus?: boolean focusArg?: string + /** Identifies the step whose agent form this one accompanies, so it can offer the fields that + * form has open. Same key `AiAgentStepInputs` is given. */ + openFieldsKey?: string + /** Fields to offer whatever the step holds, for a surface where nothing else can set them + * (`AGENT_EDITOR_RUN_INPUTS`). */ + runInputKeys?: readonly string[] } let { @@ -31,7 +38,9 @@ pickableProperties, isValid = $bindable(true), autofocus = false, - focusArg = undefined + focusArg = undefined, + openFieldsKey = undefined, + runInputKeys = [] }: Props = $props() const { stepsInputArgs, flowStateStore, flowStore, previewArgs, opWorkspace } = @@ -46,10 +55,11 @@ /** An agent asks for the same fields here that its own form shows: a setting the step leaves * unset is not something a run needs told, and listing all eleven buries the message under the - * configuration. What the step configures stays, as it does on any other step. A schema key the - * field registry doesn't know is kept, so a new one is never silently dropped. A run input is - * kept whatever the step holds: this form has no add-field control, so hiding one would leave - * no way at all to supply it. */ + * configuration. What the step configures stays, as it does on any other step, along with the + * rows its form has open — a field added there and left at its default reads as unset from the + * transforms alone, and this form has no add-field control to get it back. `runInputKeys` is + * for a surface whose form cannot open a row at all. A schema key the field registry doesn't + * know is kept, so a new one is never silently dropped. */ let schemaKeys = $derived(Object.keys(schema?.properties ?? {})) let visibleKeys = $derived.by(() => { @@ -58,7 +68,9 @@ const transforms = (mod.value as { input_transforms?: Record }) ?.input_transforms const visible = initialVisibleAgentFields(transforms, schema?.properties) - const known = new Set(AGENT_FIELDS.filter((f) => !f.runInput).map((f) => f.key)) + for (const key of openAgentFields(openFieldsKey)) visible.add(key) + for (const key of runInputKeys) visible.add(key) + const known = new Set(AGENT_FIELDS.map((f) => f.key)) return all.filter((key) => !known.has(key) || visible.has(key)) }) diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index b4e82ced2f..64a318acc7 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -14,6 +14,14 @@ import JobLoader, { type Callbacks } from './JobLoader.svelte' import { getStepHistoryLoaderContext } from './stepHistoryLoader.svelte' import { loadSchemaFromModule } from './flows/flowInfers' + import { + inlineAgentDraft, + loadLinkedAgentDrafts, + normalizeAgentRef, + type LinkedAgentDraft + } from './flows/linkedAgentDrafts' + import { AGENT_FLOW_LOCAL_KEYS } from './flows/agentResourceUtils' + import { sendUserToast } from '$lib/toast' interface Props { mod: FlowModule @@ -138,7 +146,35 @@ } else if (val.type == 'aiagent') { const { schema } = await loadSchemaFromModule(mod, opWs) - const agentVal = val + // A linked step whose agent has an unsaved draft is tested as the draft, the same way the + // whole-flow preview and the agent editor's own test pane run it. `inlineAgentDraft` + // clears `agent` and moves the draft's brain and tools onto the step, so the branches + // below then treat it as a standalone agent. + let draft: LinkedAgentDraft | undefined + if (val.agent) { + const linked = normalizeAgentRef(val.agent) + try { + draft = (await loadLinkedAgentDrafts([linked], opWs)).get(linked) + } catch (err: any) { + // The load refuses when the agent's unsaved changes cannot be read, and this function's + // caller neither awaits nor catches: without this the rejection is unhandled and the + // button appears to do nothing, with the test already marked as started. + sendUserToast(`Could not run test: ${err?.body ?? err}`, true) + // Guarded like every other access to it here: the entry is only created for steps the + // panel is tracking, and this runs on a path where it may never have been. + if (modulesTestStates.states[mod.id]) { + modulesTestStates.states[mod.id].loading = false + } + return + } + } + const agentVal = draft ? inlineAgentDraft(val, draft.args) : val + + // `args` is built from the whole AI agent schema whatever the step is, so on a linked step + // it carries every brain key as undefined even though the form renders only the flow-local + // ones (`flowLocalAgentSchema`). Overlaying those would shadow the brain the draft just + // supplied with nothing, so an inlined step takes only the inputs its form actually offers. + const formKeys = draft ? (AGENT_FLOW_LOCAL_KEYS as readonly string[]) : Object.keys(args) // The test form only covers the schema it was given, and for a standalone agent that may be // the flow-local one (the agent editor shows the brain in its own form, not here). Take the @@ -150,7 +186,7 @@ ? {} : ((agentVal.input_transforms ?? {}) as Record)), ...Object.fromEntries( - Object.keys(args).map((key) => [ + formKeys.map((key) => [ key, { expr: `flow_input.${key}`, diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index 14658a4228..9ddd7f7fdc 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -1,3 +1,42 @@ + +
    - {#if loading} + {#snippet action()} + + {#snippet children({ item })} + {#each Object.keys(WINDOWS) as key (key)} + + {/each} + {/snippet} + + {/snippet} + + {#if metrics.error} + {metrics.error.message} + {:else if metrics.current === undefined} - {:else if noMetrics} -

    No jobs delayed by more than 3 seconds in the last 14 days

    + {:else if metrics.current.tags.length === 0} +

    + No jobs delayed by more than 3 seconds in the last {WINDOWS[windowKey].label} +

    {:else}
    - {#if countData} - 3s)' - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - y: { - title: { - display: true, - text: 'count' - } - } - } - }} - /> - {/if} - {#if delayData} - 3s)' - }, - tooltip: { - callbacks: { - label: function (context) { - // @ts-ignore - if (context.raw.y === 1) { - return context.dataset.label + ': 0' - } else { - // @ts-ignore - return context.dataset.label + ': ' + context.raw.y - } - } - } - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - - y: { - type: 'logarithmic', - title: { - display: true, - text: 'delay (s)' - }, - ticks: { - callback: (value, _) => (value === 1 ? '0' : value) - } - } - } - }} - /> - {/if} + + - Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are - included in the graph. + Only tags with jobs delayed by more than 3 seconds in this window are included. At wide + windows a line shows the highest value of each time slot, so short spikes stay visible.
    {/if} diff --git a/frontend/src/lib/components/QueueStatusTable.svelte b/frontend/src/lib/components/QueueStatusTable.svelte new file mode 100644 index 0000000000..5f34e80c60 --- /dev/null +++ b/frontend/src/lib/components/QueueStatusTable.svelte @@ -0,0 +1,115 @@ + + +
    + {#snippet action()} +
+ Tag + Waiting + Next job's wait + Running + Workers + + + + {#each rows as s (s.tag)} + + {s.tag} + {s.waiting} + + {s.delay === undefined ? '-' : msToReadableTime(s.delay * 1000, 0)} + + {s.running} + + {#if unserved(s)} + + + No worker currently pulls this tag + + {:else} + {s.workers} + {/if} + + + {/each} + + + {/if} + diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 89acac79de..bcb886ee7d 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -1,19 +1,27 @@ - - diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 03b410d232..cac8952706 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -84,6 +84,8 @@ initialPath?: string } + let { initialPath }: Props = $props() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -100,25 +102,30 @@ let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean') let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean') - let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema)) + function filterSeeds() { + return { + path: initialPath || undefined, + job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined, + show_future_jobs: showFutureJobsStorage.val === false ? false : undefined + } + } - let { initialPath }: Props = $props() + let filters = useUrlSyncedFilterInstance( + untrack(() => runsFilterSearchbarSchema), + untrack(filterSeeds) + ) + + // `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own + // "Runs" entry, `/runs/` → `/runs`, Back — rewrites the query without remounting, and + // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, + // which never reaches `page.url`, so a filter the user clears stays cleared. + $effect(() => { + page.url.href + untrack(() => filters.seed(filterSeeds())) + }) let batchRerunOptionsIsOpen = $state(false) - // Initialize path filter from route param if provided and not already set via query params - if (untrack(() => initialPath) && !filters.val.path) { - filters.val.path = untrack(() => initialPath) - } - - // Apply persistent toggle values from local storage if URL doesn't specify them - if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) { - filters.val.job_trigger_kind = '!schedule' - } - if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) { - filters.val.show_future_jobs = false - } - // Sync toggle state back to local storage when filters change $effect(() => { if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') { diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index e069fe50b8..393744dbca 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -102,7 +102,7 @@ import type { SavedAndModifiedValue } from './common/confirmationModal/unsavedTypes' import DeployButton from './DeployButton.svelte' import { type Trigger, deployTriggers, handleSelectTriggerFromKind } from './triggers/utils' - import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' + import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte' import { Triggers } from './triggers/triggers.svelte' import type { ScriptBuilderProps } from './script_builder' import WorkerTagSelect from './WorkerTagSelect.svelte' @@ -1149,7 +1149,7 @@ currentValue={script} /> - t.draftConfig)} on:canceled={() => { diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index fdab141d03..f8028aeb1f 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -15,7 +15,7 @@ import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils' import Editor from './Editor.svelte' import { inferArgs, inferAssets, inferAnsibleExecutionMode } from '$lib/infer' - import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + import { injectPartitionArg } from '$lib/scriptEditorSchema' import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow' import WacDiagram from '$lib/components/graph/WacDiagram.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -1074,55 +1074,6 @@ } } - // A `// partitioned` pipeline script is materialized one slice at a time and - // receives the slice as a runtime `partition` arg (the cascade injects it in - // production). It isn't a code parameter, so schema inference doesn't see it — - // surface it in the test form so a partitioned script can be run manually. - function injectPartitionArg( - s: any, - a: Record | undefined, - l: string | undefined, - c: string - ) { - try { - if (l !== 'duckdb' || !s?.properties) return - const part = parsePipelineAnnotations(c).partition - if (!part) return - // Date-based partition kinds render a date / datetime picker; a dynamic - // key is a free-form string. - const format = - part.kind === 'hourly' - ? 'date-time' - : part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly' - ? 'date' - : undefined - if (!s.properties['partition']) { - s.properties['partition'] = { - type: 'string', - ...(format ? { format } : {}), - // ISO output so partition keys sort lexicographically (the date - // picker defaults to dd-MM-yyyy otherwise). - ...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}), - description: - part.kind === 'dynamic' - ? 'Partition key value to materialize.' - : `Partition (${part.kind}) to materialize.` - } - if (Array.isArray(s.order) && !s.order.includes('partition')) { - s.order = ['partition', ...s.order] - } - } - // Pre-fill the *test* arg with the current slice for date kinds — a - // convenience default, kept on the args (not baked into the schema, - // where it would persist to the deployed script and go stale). - if (format && a && (a['partition'] == null || a['partition'] === '')) { - const now = new Date() - a['partition'] = - format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16) - } - } catch (e) {} - } - async function inferModuleSchema() { if (activeModuleTab === null) return try { diff --git a/frontend/src/lib/components/SuperadminSettings.svelte b/frontend/src/lib/components/SuperadminSettings.svelte index 316a6237cd..dfe20ae054 100644 --- a/frontend/src/lib/components/SuperadminSettings.svelte +++ b/frontend/src/lib/components/SuperadminSettings.svelte @@ -89,7 +89,7 @@ } - + {#snippet titleExtra()} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 97b15fe443..0fd7991597 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -26,11 +26,10 @@ CheckCircle2, ExternalLink, Pencil, + Settings, UserMinus, UserPlus } from 'lucide-svelte' - import Badge from './common/badge/Badge.svelte' - import Tooltip from './Tooltip.svelte' import DropdownV2 from './DropdownV2.svelte' import Popover from './meltComponents/Popover.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -155,6 +154,11 @@ loadExtJwtPage(1) let tab: string = $state('users') + let usersListShown = $derived( + tab === 'users' && + !yamlMode && + (usersSubTab === 'users' || (usersSubTab === 'ext_jwt' && extJwtTokens.length === 0)) + ) $effect(() => { tab = $instanceSettingsSelectedTab @@ -320,11 +324,14 @@
-
+ +
{#if tab === 'ai' && !yamlMode} {:else if tab === 'users' && !yamlMode} -
+
{#if !automateUsernameCreation && !isCloudHosted()}

Automatic username creation

@@ -373,7 +380,7 @@ - {#if usersSubTab === 'users' || (usersSubTab === 'ext_jwt' && extJwtTokens.length === 0)} + {#if usersListShown} {filteredUsers.length} user{filteredUsers.length !== 1 ? 's' : ''} found

-
- 50} - loadMore={50} - on:loadMore={() => { - nbDisplayed += 50 - }} - > + +
+
Email @@ -434,7 +437,7 @@ Kind {/if} Role - + Actions @@ -443,12 +446,22 @@ {#if filteredUsers && users} {#each filteredUsers.slice(0, nbDisplayed) as { email, super_admin, devops, login_type, name, username, operator_only, is_workspace_admin, role_source, disabled, workspace_id }, i (email + '::' + (workspace_id ?? ''))} {@const isServiceAccount = login_type === 'service_account'} + {@const groupRole = + role_source === 'instance_group' && (super_admin || devops)} + + {@const groupRoleTooltip = + 'Role is set by an instance group. Superadmin and Devops can be set here, but demoting to User requires removing the user from the group.'} + {@const serviceAccountTooltip = + 'Service accounts are always users in the instance. Their workspace role is managed in the workspace user settings.'} + - +
{#if isServiceAccount} @@ -458,14 +471,6 @@ >{email} {/if} - {#if workspace_id} - - {truncate(workspace_id, 20)} - - {/if} {#if disabled} {#if automateUsernameCreation} - + {#if username} {username} {:else} @@ -503,133 +508,157 @@ > {#if activeOnly} - {#if is_workspace_admin} - Admin - {:else if operator_only} - Operator only - {:else} - Developer - {/if} + + {#if is_workspace_admin} + Admin + {:else if operator_only} + Operator only + {:else} + Developer + {/if} + {/if} - {#if isServiceAccount} -
+ +
+ {#key `${super_admin}_${devops}_${role_source}`} + { + if (email == $userStore?.email) { + sendUserToast('You cannot demote yourself', true) + listUsers(activeOnly) + return + } + + let role = e.detail + + if (role === 'super_admin') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: true, + is_devops: false + } + }) + } + if (role === 'devops') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: false, + is_devops: true + } + }) + } + if (role === 'user') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: false, + is_devops: false + } + }) + } + sendUserToast('User updated') + listUsers(activeOnly) + }} + > + {#snippet children({ item })} + + + + {/snippet} + + {/key} + {#if isServiceAccount} {is_workspace_admin ? 'Admin' : operator_only ? 'Operator' : 'Developer'} + in + {#if workspace_id} + closeDrawer?.()} + >{truncate(workspace_id, 20)} + {:else} + its workspace + {/if} - - Service-account role is managed in the workspace user settings. - -
- {:else} -
- {#key `${super_admin}_${devops}_${role_source}`} - { - if (email == $userStore?.email) { - sendUserToast('You cannot demote yourself', true) - listUsers(activeOnly) - return - } - - let role = e.detail - - if (role === 'super_admin') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: true, - is_devops: false - } - }) - } - if (role === 'devops') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: false, - is_devops: true - } - }) - } - if (role === 'user') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: false, - is_devops: false - } - }) - } - sendUserToast('User updated') - listUsers(activeOnly) - }} - > - {#snippet children({ item })} - - - - {/snippet} - - {/key} - {#if role_source === 'instance_group' && (super_admin || devops)} - closeDrawer?.()} - > - Set by instance group - - {/if} -
- {/if} + {:else if groupRole} + closeDrawer?.()} + > + Set by instance group + + {/if} +
- +
{#if isServiceAccount} {#if workspace_id} - Manage in workspace + closeDrawer?.(), + href: `${base}/workspace_settings?tab=users&workspace=${workspace_id}` + } + ]} + /> {/if} {:else}
{/each} + {#if filteredUsers.length > nbDisplayed} + {@const remaining = Math.min(50, filteredUsers.length - nbDisplayed)} + + + + + + + {/if} {/if} diff --git a/frontend/src/lib/components/TimeAgo.svelte b/frontend/src/lib/components/TimeAgo.svelte index cf3f019432..6138cdb6da 100644 --- a/frontend/src/lib/components/TimeAgo.svelte +++ b/frontend/src/lib/components/TimeAgo.svelte @@ -27,18 +27,16 @@ let interval onMount(() => { - // compact schedules itself below; it needs no fixed rate. - if (compact) return + // compact and noSeconds schedule themselves below; they need no fixed rate. + if (compact || noSeconds) return - // Update every minute for noSeconds mode, every second otherwise. - const intervalMs = noSeconds ? 60000 : 1000 interval = setInterval(() => { computeDate() if (!isRecent) { clearInterval(interval) interval = undefined } - }, intervalMs) + }, 1000) // Add explicit cleanup return () => { @@ -49,21 +47,27 @@ // Waking on the boundary of the unit on screen, rather than at a fixed rate: `2h` only // changes on the hour, and a row that reads `5d` must not hold a 1s timer to find that // out. Re-armed when `date` changes, so an item edited to now leaves its day-long wait. + // noSeconds rides the same schedule: its relative forms keep counting hours and days, + // so a fixed-rate timer that stops once the date is an hour old would freeze them. $effect(() => { - if (!compact) return + if (!compact && !noSeconds) return const at = date + // An absent or unparsable date has no boundary to wait for: the delay below would + // be NaN, which setTimeout runs immediately, and the tick would re-arm itself in + // a tight loop. + if (Number.isNaN(new Date(at).getTime())) return let handle: ReturnType | undefined const tick = () => { computeDate() - handle = setTimeout(tick, compactDelayMs(at)) + handle = setTimeout(tick, nextUnitBoundaryMs(at)) } - handle = setTimeout(tick, compactDelayMs(at)) + handle = setTimeout(tick, nextUnitBoundaryMs(at)) return () => { handle && clearTimeout(handle) } }) - function compactDelayMs(dateString: string): number { + function nextUnitBoundaryMs(dateString: string): number { const secs = secondsAgo(new Date(dateString)) const left = secs < 60 diff --git a/frontend/src/lib/components/VariableEditor.svelte b/frontend/src/lib/components/VariableEditor.svelte index 14082bb901..7c935be45c 100644 --- a/frontend/src/lib/components/VariableEditor.svelte +++ b/frontend/src/lib/components/VariableEditor.svelte @@ -1,7 +1,7 @@ - - diff --git a/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte new file mode 100644 index 0000000000..a1a58a4ce5 --- /dev/null +++ b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte @@ -0,0 +1,219 @@ + + +
+
+
+ {#if context} + + {context} + + {/if} + {#if job} +

+ Requested by {job.created_by} · +

+ {/if} +
+ {STATUS_BADGE[status].label} +
+ + {#if typeof approvalInfo.description === 'string'} +

{approvalInfo.description}

+ {:else if approvalInfo.description != undefined} + + {/if} + + {#if status === 'pending'} + {#if hasForm} + {#if emptyString($enterpriseLicense)} + + {:else} + + {/if} + {/if} + + {#if approvalInfo.can_approve} +
+ {#if approvalInfo.hide_cancel !== true} + + {:else} +
+ {/if} + +
+ {#if isSelfApprovalBypass} + + As an administrator, by approving or rejecting this request, you bypass the self-approval + interdiction. + + {/if} + {:else if approvalInfo.user_auth_required && !$userStore} +

Sign in to review this request.

+ + {:else} +
+

You are not authorized to approve this request.

+ {#if isSelfApprovalRefused} +

Self-approval is disabled for this step.

+ {/if} + {#if groupsRequired.length > 0} +

+ Only members of the following groups can approve: + {groupsRequired.join(', ')} +

+ {/if} +
+ {/if} + {:else} +
+ {#if status === 'approved'} + + {:else if status === 'rejected'} + + {:else} + + {/if} +
+ + {status === 'closed' ? 'This request is closed' : STATUS_BADGE[status].label} + + + {#if status === 'approved'} + Your approval was recorded. You can close this page. + {:else if status === 'rejected'} + Your rejection was recorded. You can close this page. + {:else} + The flow is no longer waiting for approval. + {/if} + +
+
+ {/if} + + {#if !isLocked && ((status === 'pending' && approvalInfo.approvers.length > 0) || isWorkspaceMember)} +
+ + {#if status === 'pending' && approvalInfo.approvers.length > 0} + Already approved by {approvalInfo.approvers.map((a) => a.approver).join(', ')} + {/if} + + {#if isWorkspaceMember} + + {/if} +
+ {/if} +
diff --git a/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts new file mode 100644 index 0000000000..1dec5e9da8 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { renderDbEqualityFilter, renderDbLiteral } from './utils' + +describe('renderDbLiteral', () => { + it('doubles single quotes on every dialect', () => { + expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'") + expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'") + }) + + it('doubles backslashes only where the dialect treats them as escapes', () => { + expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'") + expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'") + expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'") + }) + + it('marks SQL Server strings as Unicode constants', () => { + expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'") + }) + + it('renders numbers and booleans without quotes', () => { + expect(renderDbLiteral(42, 'postgresql')).toBe('42') + expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE') + expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1') + }) + + it('has no literal for values that cannot be compared safely', () => { + expect(renderDbLiteral(null, 'postgresql')).toBeUndefined() + expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined() + expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined() + }) +}) + +describe('renderDbEqualityFilter', () => { + it('quotes the identifier per dialect', () => { + expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'") + expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined() + }) + + it('doubles a delimiter embedded in the identifier', () => { + expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`) + expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1') + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index bf81ec07bb..c0477e8408 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -333,25 +333,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string { .join(', ') } +/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an + * embedded delimiter. */ export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string { switch (dbType) { case 'postgresql': - return `"${identifier}"` // PostgreSQL uses double quotes for identifiers - case 'ms_sql_server': - return `[${identifier}]` // MSSQL uses square brackets for identifiers - case 'mysql': - return `\`${identifier}\`` // MySQL uses backticks case 'snowflake': - return `"${identifier}"` // Snowflake uses double quotes for identifiers - case 'bigquery': - return `\`${identifier}\`` // BigQuery uses backticks case 'duckdb': - return `"${identifier}"` // DuckDB uses double quotes for identifiers + return `"${identifier.replace(/"/g, '""')}"` + case 'ms_sql_server': + return `[${identifier.replace(/]/g, ']]')}]` + case 'mysql': + case 'bigquery': + return `\`${identifier.replace(/`/g, '``')}\`` default: throw new Error('Unsupported database type: ' + dbType) } } +/** Renders a cell value as a SQL literal. Returns undefined for values that + * have no safe literal form (null, objects, non-finite numbers). */ +export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined { + if (value === null || value === undefined) return undefined + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') { + if (dbType === 'ms_sql_server') return value ? '1' : '0' + return value ? 'TRUE' : 'FALSE' + } + if (typeof value !== 'string') return undefined + let escaped = value.replace(/'/g, "''") + // MySQL, Snowflake and BigQuery treat a backslash inside a string literal as + // an escape character. + if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') { + escaped = escaped.replace(/\\/g, '\\\\') + } + // A plain constant is varchar on SQL Server and goes through the database + // code page; the N prefix keeps it Unicode against nvarchar columns. + return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'` +} + +/** `"column" = ` predicate, or undefined when the value can't be + * rendered as a literal. */ +export function renderDbEqualityFilter( + column: string, + value: unknown, + dbType: DbType +): string | undefined { + const literal = renderDbLiteral(value, dbType) + if (literal === undefined) return undefined + return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}` +} + export function getLanguageByResourceType(name: string): ScriptLang { const language = { postgresql: 'postgresql', diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 8ef4ffbf95..565ac5d5d3 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -473,15 +473,6 @@ let appEditorHeader: AppEditorHeader | undefined = $state(undefined) - export function triggerTutorial() { - const urlParams = new URLSearchParams(window.location.search) - const tutorial = urlParams.get('tutorial') - - if (tutorial) { - appEditorHeader?.runTutorialById(tutorial) - } - } - let box: HTMLElement | undefined = $state(undefined) function parseScroll() { $yTop = box?.scrollTop ?? 0 diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index a57dc87622..ab043f81a1 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -7,25 +7,13 @@ import { redo, undo } from '$lib/history.svelte' import { discardDraftAfterDeploy } from '$lib/userDraftToast' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' - import { - enterpriseLicense, - tutorialsToDo, - userStore, - userWorkspaces, - workspaceStore - } from '$lib/stores' + import { enterpriseLicense, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { isMac, type Item, userPathPrefix } from '$lib/utils' - import { resetAllTodos, skipAllTodos } from '$lib/tutorialUtils' - import { getTutorialIndex } from '$lib/tutorials/config' import { random_adj } from '$lib/components/random_positive_adjetive' import { AlignHorizontalSpaceAround, BellOff, - BookOpen, Bug, - CheckCheck, - CheckCircle, - Circle, DiffIcon, Expand, FileJson, @@ -33,7 +21,6 @@ FormInput, History, Laptop2, - RefreshCw, Save, Smartphone, FileClock, @@ -61,7 +48,6 @@ import Awareness from '$lib/components/Awareness.svelte' import { secondaryMenuLeftStore, secondaryMenuRightStore } from './settingsPanel/secondaryMenu' import Dropdown from '$lib/components/DropdownV2.svelte' - import AppEditorTutorial from './AppEditorTutorial.svelte' import AppReportsDrawer from './AppReportsDrawer.svelte' import DebugPanel from './contextPanel/DebugPanel.svelte' @@ -679,49 +665,9 @@ action: () => { appExport?.open(toStatic($app, $staticExporter, $summary).app) } - }, - { - displayName: 'Tutorials', - icon: BookOpen, - separatorTop: true, - submenuItems: [ - { - displayName: 'Background runnables', - action: () => appEditorTutorial?.runTutorialById('backgroundrunnables'), - icon: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? Circle - : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? undefined - : 'green' - }, - { - displayName: 'Connection', - action: () => appEditorTutorial?.runTutorialById('connection'), - icon: $tutorialsToDo.includes(getTutorialIndex('connection')) ? Circle : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('connection')) ? undefined : 'green' - }, - { - displayName: 'Reset tutorials', - action: () => resetAllTodos(), - icon: RefreshCw, - separatorTop: true - }, - { - displayName: 'Skip tutorials', - action: () => skipAllTodos(), - icon: CheckCheck - } - ] } ]) as Item[] - let appEditorTutorial: AppEditorTutorial | undefined = $state(undefined) - - export function runTutorialById(id: string, options?: { skipStepsCount?: number }) { - appEditorTutorial?.runTutorialById(id, options) - } - let appReportingDrawerOpen = $state(false) export function openTroubleshootPanel() { @@ -1090,15 +1036,7 @@ {/if}
-
- - {#if $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) || $tutorialsToDo.includes(getTutorialIndex('connection'))} - - {/if} -
- +
{#if hasErrors} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index 7b68389d82..836af68db6 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -115,6 +115,10 @@ // letting the publisher believe the app is open. let guestAccessEnabled: boolean | undefined = $state(undefined) let guestUsage: GuestUsage | undefined = $state(undefined) + // Whether the deployment can have guests at all; off, the mode is not on offer. The + // backend decides; the hostname stands in until it has answered, the shared cloud + // being the only deployment where guests are unavailable. + let guestsAvailable = $derived.by(() => guestUsage?.available ?? !isCloudHosted()) $effect(() => { const ws = opWs @@ -458,7 +462,7 @@
{/if} - {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest'} + {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest' && guestsAvailable} Opening this app to guests is restricted to workspace admins and bypass users by a workspace protection rule @@ -481,8 +485,10 @@ - {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase} + {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase && guestsAvailable}
Embed for your own authenticated users (guest JWT) diff --git a/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte b/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte deleted file mode 100644 index c268058717..0000000000 --- a/frontend/src/lib/components/apps/editor/AppEditorTutorial.svelte +++ /dev/null @@ -1,35 +0,0 @@ - - -) => { - targetTutorial = event.detail.detail - }} -/> - - { - targetTutorial = undefined - }} - on:confirmed={async () => { - window.open(`/apps/add?tutorial=${targetTutorial}`, '_blank') - }} -> -
- This tutorial can only be run on a new app. -
- diff --git a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte index 4ae67dbb0e..614d1f4fa4 100644 --- a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte +++ b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte @@ -9,10 +9,10 @@ cleanValueProperties, displayDate, emptyString, - orderedYamlStringify, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AppService, type AppWithLastVersion, type AppHistory } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { Skeleton } from '$lib/components/common' diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index ce6461a46e..6f69b36400 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -23,14 +23,11 @@ let { workspace, path, - editHref, onEdit, onLoadState }: { workspace: string path: string - /** Where the Edit button points (low-code vs raw editor). */ - editHref: string /** Handle Edit in place instead of following `editHref`. An AI session shows * this viewer inside a preview tab, where a plain link would navigate the whole * page out of the session rather than flipping the tab to its editor. */ @@ -44,6 +41,12 @@ let notExists = $state(false) let noPermission = $state(false) let canWriteApp = $state(false) + /** Raw vs low-code, read from the app itself rather than from the route: + * both kinds render here and either route serves either kind (links to a raw + * app point at /apps/get all over the app), so only the app can say which + * editor the Edit button must open. */ + let isRawApp = $state(false) + let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined // The opaque iframe loads the dedicated cookieless, chrome-less viewer route. @@ -116,11 +119,14 @@ } } - // Edit button: determine write access on this real-origin page (cookie). + // Edit button: determine write access and which editor to open on this + // real-origin page (cookie). The sandboxed low-code app never loads on this + // page (it loads inside the opaque iframe), so `app` can't be the source. async function loadPerms() { try { const lite: any = await AppService.getAppLiteByPath({ workspace, path }) canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore) + isRawApp = !!lite?.raw_app } catch (_) { canWriteApp = false } diff --git a/frontend/src/lib/components/apps/editor/PublicApp.svelte b/frontend/src/lib/components/apps/editor/PublicApp.svelte index 6523f13ffa..6de8f9df0c 100644 --- a/frontend/src/lib/components/apps/editor/PublicApp.svelte +++ b/frontend/src/lib/components/apps/editor/PublicApp.svelte @@ -1,14 +1,14 @@ @@ -111,18 +111,6 @@ >Powered by   Windmill
- - {#snippet userInfo(child)} -
{child}
- {/snippet} - -
{#if $userStore} - {@render userInfo($userStore.username)} - {:else if globalUser} - {@render userInfo(globalUser.email)} - {:else}{/if} -
{/if} {#if notExists} @@ -188,27 +176,35 @@ )} style={app?.value?.['css']?.['app']?.['viewer']?.style} > - goto(path)} - gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} - /> + {#await loadAppPreview()} + + {:then Module} + goto(path)} + gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} + /> + {:catch} +
+ Reload the page to try again. +
+ {/await}
{/if} {/key} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index dbe8f8d503..e994767488 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -26,12 +26,14 @@ import { OpenAPI, UserService } from '$lib/gen' import { page } from '$app/state' import { onDestroy, onMount, setContext, type Snippet } from 'svelte' - import { Alert, Skeleton } from '$lib/components/common' + import Alert from '$lib/components/common/alert/Alert.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import { base } from '$app/paths' import { goto } from '$app/navigation' import Login from '$lib/components/Login.svelte' import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils' import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types' + import { loadAppPreview } from './loadAppPreview' import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte' import { hasStoredSdkConsent, storeSdkConsent } from '$lib/components/raw_apps/sdkScopes' @@ -420,7 +422,8 @@ if (unsandboxed || isRaw) { // Render the app directly on this origin: same-origin when unsandboxed // (the default), or a single opaque bundle iframe when it's a sandboxed - // raw app. + // raw app. A low-code app's runtime downloads alongside the app payload. + if (!isRaw) loadAppPreview().catch(() => {}) onViewerReady?.(undefined, requestTokenRefresh) } else { // Sandboxed low-code: hand the scoped token to the opaque viewer iframe. @@ -529,6 +532,8 @@ onMount(() => { if (isViewer) { + // Only a sandboxed low-code app is ever framed as a viewer. + loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() // Announce readiness so the embedder sends us the token. diff --git a/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte b/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte index 2ac045feaa..aa81b7f5cc 100644 --- a/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte +++ b/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte @@ -11,7 +11,6 @@ left } from './componentCallbacks.svelte' import type { AppEditorContext, AppViewerContext } from '../../types' - import { isCurrentlyInTutorial } from '$lib/stores' const { history, movingcomponents, jobsDrawerOpen, runnableJobEditorPanel } = getContext('AppEditorContext') as AppEditorContext @@ -34,8 +33,7 @@ if ( (typeof classes === 'string' && classes.includes('inputarea')) || ['INPUT', 'TEXTAREA'].includes(document.activeElement?.tagName!) || - $runnableJobEditorPanel.focused || - isCurrentlyInTutorial.val + $runnableJobEditorPanel.focused ) { return } diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte index 214ab3e31d..a4bed4affb 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte @@ -14,10 +14,9 @@ import { defaultCode } from '../component' import WorkspaceScriptList from '../settingsPanel/mainInput/WorkspaceScriptList.svelte' import RunnableSelector from '../settingsPanel/mainInput/RunnableSelector.svelte' - import { defaultScripts, isCurrentlyInTutorial } from '$lib/stores' + import { defaultScripts } from '$lib/stores' import DefaultScripts from '$lib/components/DefaultScripts.svelte' import type { Preview } from '$lib/gen' - import { twMerge } from 'tailwind-merge' import type { InlineScript } from '../../sharedTypes' interface Props { @@ -122,13 +121,7 @@ -
+
Choose a language
diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte index aaffb1feaf..56304ca3e3 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanelList.svelte @@ -7,10 +7,6 @@ import { getAllScriptNames } from '../../utils' import PanelSection from '../settingsPanel/common/PanelSection.svelte' import { getAppScripts } from './utils' - import AppTutorials from '$lib/components/AppTutorials.svelte' - import { tutorialsToDo } from '$lib/stores' - import { ignoredTutorials } from '$lib/components/tutorials/ignoredTutorials' - import { tutorialInProgress } from '$lib/tutorialUtils' import DocLink from '../settingsPanel/DocLink.svelte' import HideButton from '../settingsPanel/HideButton.svelte' import { BG_PREFIX } from '../appUtilsCore' @@ -37,10 +33,6 @@ } function createBackgroundScript() { - if ($tutorialsToDo.includes(5) && !$ignoredTutorials?.includes(5) && !tutorialInProgress()) { - appTutorials?.runTutorialById('backgroundrunnables', { skipStepsCount: 2 }) - } - for (const [index, script] of $app.hiddenInlineScripts.entries()) { if (script.hidden) { delete script.hidden @@ -75,7 +67,6 @@ selectScript(`${BG_PREFIX}${$app.hiddenInlineScripts.length - 1}`) } - let appTutorials: AppTutorials | undefined = $state(undefined) const dispatch = createEventDispatcher() let runnables = $derived(getAppScripts($app.grid, $app.subgrids)) // When selected component changes, update selectedScriptComponentId @@ -248,5 +239,3 @@
- - diff --git a/frontend/src/lib/components/apps/editor/loadAppPreview.ts b/frontend/src/lib/components/apps/editor/loadAppPreview.ts new file mode 100644 index 0000000000..57e82e09f8 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/loadAppPreview.ts @@ -0,0 +1,12 @@ +// AppPreview must stay a dynamic import: it statically reaches every low-code component +// (and through them monaco, ag-grid, chart.js...), none of which a raw app renders. +// assertLeanPublicAppRoutes in vite.config.js fails the build if that regresses. +let pending: Promise | undefined + +export function loadAppPreview() { + pending ??= import('./AppPreview.svelte').catch((e) => { + console.error('Could not load the low-code app viewer', e) + throw e + }) + return pending +} diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 2c5e5fd97e..4b252b9338 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -36,7 +36,8 @@ type ColumnLineage, type PipelineAnnotations } from './parsePipelineAnnotations' - import ColumnLineageTrace from './ColumnLineageTrace.svelte' + import ColumnTraceSection from './ColumnTraceSection.svelte' + import DbtColumnList from './DbtColumnList.svelte' import { extractDraftMacros } from './resolveGraph' import { assetColumnNodes, type ColumnLineageGraph } from './columnLineageGraph' import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte' @@ -156,6 +157,15 @@ // resolved graph). Drives the transitive column-lineage trace shown for a // selected materialized asset. selectionColumnGraph?: ColumnLineageGraph + /** That graph still being fetched — a dbt relation's lineage is a request + * of its own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** The lineage reaches past what the graph holds: the API cut it at the + * part nearest the selection. */ + selectionColumnTruncated?: boolean + /** That trace could not be fetched. Distinguished from an empty one: a + * project without the analysis pass draws nothing either. */ + selectionColumnFailed?: boolean /** dbt provenance of the selected relation, when a dbt project * materializes it — carries the model's own SQL. */ selectionDbt?: DbtAssetProvenance @@ -289,6 +299,9 @@ onScriptRemoved, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, + selectionColumnFailed = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -446,6 +459,19 @@ return scripts.length === 1 ? `${scripts[0].path}__dbt/${file}` : file }) + // The two things a dbt relation can show besides its SQL, and what decides + // whether the panel opens at all for one that has none: the columns the model + // produces, and the trace they sit in. A share-link viewer gets neither — + // both are gated on reading the project, like the SQL. + let selectionDbtHasColumns = $derived( + !!selectionDbt?.column_schema?.length || Object.keys(selectionDbt?.columns ?? {}).length > 0 + ) + let selectionColumnNodes = $derived( + selection?.kind === 'asset' && selectionColumnGraph + ? assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path) + : [] + ) + // Bound from ScriptEditor — populated by inferAssets on every code // change. Forwarded to the page so the canvas can re-derive write // edges as the user edits the body (e.g. renaming a CREATE TABLE @@ -1221,22 +1247,21 @@
{/if} - {#if selectionColumnGraph && assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path).length > 0} -
- -
- {/if} +
{/key} - {:else if selectionDbt?.raw_code} + {:else if selectionDbt && (selectionDbt.raw_code || selectionDbtHasColumns || selectionColumnNodes.length > 0 || selectionColumnLoading || selectionColumnFailed)} + {#if selectionDbtHasColumns} +
+ +
+ {/if} + + {#if selectionDbt.raw_code} +
+ +
+ {/if}
{:else}
diff --git a/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte new file mode 100644 index 0000000000..28afa5904e --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte @@ -0,0 +1,73 @@ + + + +{#snippet failure()} +
+ Part of this column lineage could not be loaded, so the trace may be incomplete. +
+{/snippet} + +{#if loading && nodes.length === 0} +
+ + Loading column lineage +
+{:else if nodes.length === 0} + {#if failed} +
{@render failure()}
+ {/if} +{:else if graph} +
+ + {#if failed} + {@render failure()} + {/if} + {#if truncated} +
+ Showing the part of the trace nearest this relation. The lineage reaches further than one + view can draw. +
+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte new file mode 100644 index 0000000000..a1c769d9af --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte @@ -0,0 +1,50 @@ + + +{#if columns.length > 0} +
+
{analyzed ? 'columns' : 'columns declared'}
+
+ {#each columns as col (col.name)} +
+ {col.name} + {#if col.type} + {col.type} + {/if} + {col.description} +
+ {/each} +
+ + {#if !analyzed} +
+ Declared metadata. Set `column_lineage: true` in the descriptor for the real column schema, + typed and in the order the model produces it. +
+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 3439fa5ad5..b71a51ea0a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -17,7 +17,9 @@ AssetGraphResponse, AssetGraphSelection, NativeTriggerKind, - PipelineMode, DbtAssetProvenance } from './types' + PipelineMode, + DbtAssetProvenance + } from './types' import type { AssetKind, Script, ScriptLang } from '$lib/gen' import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte' import type { PipelineOutputKind } from './pipelineTemplates' @@ -76,6 +78,9 @@ localScriptsVersion, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, + selectionColumnFailed = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -180,8 +185,16 @@ * the selected node's source on live-reload. */ localScriptsVersion?: unknown selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> - /** Transitive column-lineage trace for a selected ducklake asset (route page). */ + /** Transitive column-lineage trace for the selected asset (route page). */ selectionColumnGraph?: ColumnLineageGraph + /** That trace still being fetched — a dbt relation's is a request of its + * own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** That trace cut at the part nearest the selection. */ + selectionColumnTruncated?: boolean + /** That trace could not be fetched. Distinguished from an empty one: a + * project without the analysis pass draws nothing either. */ + selectionColumnFailed?: boolean /** dbt provenance of the selected relation — carries its SQL. */ selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean @@ -514,6 +527,9 @@ selection={activeDraft ? undefined : editor.selection} selectionProducers={activeDraft ? [] : selectionProducers} {selectionColumnGraph} + {selectionColumnLoading} + {selectionColumnTruncated} + {selectionColumnFailed} {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts index 6842dcfd86..16c8713c5c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import type { AssetGraphResponse } from './types' import { buildColumnGraph, + buildDbtColumnGraph, colNodeId, + mergeColumnGraphs, + type ColumnLineageGraph, traceColumn, connectedComponent, assetColumnNodes, @@ -120,6 +123,79 @@ describe('buildColumnGraph', () => { }) }) +describe('buildDbtColumnGraph', () => { + it('takes the direct kinds and drops any other', () => { + // `scan` means the column was read to produce the ROW — a join key, a + // predicate, a `group by` — so it reaches every output column of its model + // and is not what a column trace means. The server filters it out; this + // filters again, because the kind set is the engine's and an unknown one + // must not become an edge the trace calls data flow. + const g = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw_name', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'mod' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'id', + kind: 'copy' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'scan' + } + ]) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'clean_name'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw_name')]) + ) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'id'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'id')]) + ) + }) +}) + +describe('mergeColumnGraphs', () => { + it('chains a dbt column into what a producer derives from it', () => { + // The two halves arrive separately — the producer's from the asset graph, + // dbt's from its own request — and meet at the dbt node a `// column` + // annotation names. A trace has to cross that, or a dbt selection stops + // before the script consuming it. + const dbt = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw', + to_asset_path: 'main/s/mart', + to_column: 'clean', + kind: 'copy' + } + ]) + const producer: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() + } + const src = colNodeId('dbt', 'main/s/mart', 'clean') + const out = colNodeId('ducklake', 'wh/report', 'total') + producer.nodes.set(src, { kind: 'dbt', path: 'main/s/mart', column: 'clean' }) + producer.nodes.set(out, { kind: 'ducklake', path: 'wh/report', column: 'total' }) + producer.up.set(out, new Set([src])) + producer.down.set(src, new Set([out])) + + const merged = mergeColumnGraphs(dbt, producer) + expect(traceColumn(colNodeId('dbt', 'main/s/stg', 'raw'), merged)).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw'), src, out]) + ) + }) +}) + describe('traceColumn', () => { it('returns the full upstream + downstream impact set of a source column', () => { const g = buildColumnGraph(chainGraph()) diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts index 50fa1c4e7b..54837753e1 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts @@ -1,6 +1,11 @@ -import type { AssetKind } from '$lib/gen' +import type { AssetKind, DbtColumnLineage } from '$lib/gen' import type { AssetGraphResponse } from './types' +// One column-to-column edge of a dbt project's static analysis, as the API +// serves it. Taken from the generated client rather than restated: unlike the +// asset graph, this response is fetched through it. +export type DbtColumnEdge = DbtColumnLineage['edges'][number] + // A node in the column-level lineage graph: one column of one asset. export type ColumnNode = { kind: AssetKind; path: string; column: string } export type ColumnNodeId = string @@ -23,6 +28,21 @@ export type ColumnLineageGraph = { down: Map> } +export const EMPTY_COLUMN_GRAPH: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() +} + +// Direct value flow, as dbt's static analysis labels it: `copy` passes a column +// through, `mod` transforms it. The API serves only those two — the third kind, +// `scan`, means the column was read to produce the ROW rather than the value (a +// join key, a `where` predicate, a `group by`), so it reaches EVERY output +// column of the model and would draw the diagram as a complete bipartite graph. +// Filtered here as well so a kind the engine invents cannot silently become an +// edge the trace claims is data flow. +const DIRECT_DBT_LINEAGE = new Set(['copy', 'mod']) + // Build the column graph from a resolved asset graph. A producer's // `column_lineage` describes the columns of the asset it materializes; that // output asset is the ducklake target it writes (v1 materialize target), found @@ -89,6 +109,59 @@ export function buildColumnGraph(graph: AssetGraphResponse): ColumnLineageGraph return { nodes, up, down } } +// The same graph, from dbt's own column lineage. dbt arrives already resolved to +// two relations rather than anchored to a producer, and the API serves only the +// direct kinds, so this is a straight edge list. +export function buildDbtColumnGraph(edges: DbtColumnEdge[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + const addNode = (n: ColumnNode): ColumnNodeId => { + const id = colNodeId(n.kind, n.path, n.column) + if (!nodes.has(id)) nodes.set(id, n) + return id + } + for (const e of edges) { + // Belt and braces: the API filters to `copy`/`mod`, and a kind an engine + // invents must not silently become an edge the trace calls data flow. + if (!DIRECT_DBT_LINEAGE.has(e.kind)) continue + const src = addNode({ kind: 'dbt', path: e.from_asset_path, column: e.from_column }) + const out = addNode({ kind: 'dbt', path: e.to_asset_path, column: e.to_column }) + if (src === out) continue + ;(up.get(out) ?? up.set(out, new Set()).get(out)!).add(src) + ;(down.get(src) ?? down.set(src, new Set()).get(src)!).add(out) + } + return { nodes, up, down } +} + +// One graph out of several, so a trace crosses the boundary between them. +// +// The two halves reach each other through shared node ids: a producer's +// `// column out <- dbt://wh/schema/model.col` puts a `('dbt', path, column)` +// node in the producer graph under the same `colNodeId` the dbt lineage mints +// for it, so the union chains a dbt model's columns into the script that +// consumes them and on into what that script writes. Kept separate up to here +// because they are fetched separately — the producer half rides on the asset +// graph, the dbt half is asked for per selection. +export function mergeColumnGraphs(...graphs: ColumnLineageGraph[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + for (const g of graphs) { + for (const [id, n] of g.nodes) if (!nodes.has(id)) nodes.set(id, n) + for (const [dir, into] of [ + [g.up, up], + [g.down, down] + ] as const) { + for (const [id, adj] of dir) { + const target = into.get(id) ?? into.set(id, new Set()).get(id)! + for (const m of adj) target.add(m) + } + } + } + return { nodes, up, down } +} + // Every node reachable from `start` by following `adj` (transitive closure, // excluding `start` itself). Iterative to avoid deep-recursion limits. function reach(start: ColumnNodeId, adj: Map>): Set { diff --git a/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts new file mode 100644 index 0000000000..477ed630b1 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts @@ -0,0 +1,153 @@ +import { AssetService, JobService, type DbtColumnLineage } from '$lib/gen' +import { + buildDbtColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph +} from './columnLineageGraph' + +/** Which stored dbt graph a view is drawing. A job — a run's snapshot, or the + * editor's parse of its own buffer — is asked through the job route, the only + * way to reach a graph that names no deployed version; otherwise the deployed + * version by hash, or the current one when there is no hash. */ +export type DbtGraphPin = { jobId?: string; scriptHash?: string | number } + +/** What a selection's dbt column lineage is doing right now. `loading` is + * separate because a project still being fetched and one that never asked for + * the analysis pass are the same empty graph otherwise. */ +export type DbtColumnLineageState = { + readonly graph: ColumnLineageGraph + readonly loading: boolean + /** The component reaches past what `graph` holds — the API cut it at the + * part nearest the selection. */ + readonly truncated: boolean + /** The request failed, so `graph` is empty for a reason that is not "this + * project has no column lineage". */ + readonly failed: boolean +} + +function fetchLineage( + workspace: string, + assetPaths: string[], + pin: DbtGraphPin | undefined +): Promise { + return pin?.jobId + ? JobService.getDbtRunColumnLineage({ workspace, id: pin.jobId, assetPath: assetPaths }) + : AssetService.getDbtColumnLineage({ + workspace, + assetPath: assetPaths, + dbtScriptHash: pin?.scriptHash != undefined ? String(pin.scriptHash) : undefined + }) +} + +/** Follow the selection, fetching the dbt column lineage it reaches. + * + * One request per selection, whatever it reaches: the API takes every relation + * at once and walks out from all of them, so there is no partial answer to hold + * on to between selections and nothing to go stale behind a redeploy. + * + * Per selection rather than off the graph response: the graph is folder-wide + * and a run page polls it, while this is drawn for one selection. It also means + * the request is never made for a project that did not opt into the analysis + * pass — the pane simply never shows the section. + */ +export function useDbtColumnLineage(args: { + workspace: () => string | undefined + /** The dbt relations to expand. The selection itself when it is one; for a + * selection of another kind, every dbt relation its own lineage reaches — + * a ducklake table can be derived from several, and expanding only the + * first would leave the rest as leaves. */ + assetPaths: () => string[] + /** The graph on screen, so the lineage describes the same project. */ + pin?: () => DbtGraphPin | undefined + /** Which fetch of that graph is on screen. It changes when the view goes and + * gets the graph again — a Refresh, a deploy — and asking again is the whole + * point: the relation, the pin and the seeds are all unchanged by a + * redeploy, so without this the pane would pair a freshly fetched model's + * SQL and columns with the edges of the version before it. It is also what + * retries a request that failed. */ + generation?: () => unknown +}): DbtColumnLineageState { + let graph = $state(EMPTY_COLUMN_GRAPH) + let loading = $state(false) + let truncated = $state(false) + let failed = $state(false) + + // The question the state in hand answers, and a counter deciding which answer + // is still wanted. Neither is a cache of edges: the API returns a whole + // component, so an answer is either the current selection's or nothing. + let asked: string | undefined = undefined + let latest = 0 + + $effect(() => { + const workspace = args.workspace() + const paths = [...new Set(args.assetPaths())].sort() + const pin = args.pin?.() + const question = JSON.stringify([ + workspace, + pin?.jobId, + pin?.scriptHash, + args.generation?.() ?? null, + paths + ]) + // A selection re-derived from a graph that polled is the same question. Not + // asking it again is what keeps a run page from refetching a component's + // worth of edges every poll to redraw what is already on screen — while a + // graph the view deliberately went and fetched moves `generation`, so that + // IS a new question. + if (question === asked) return + asked = question + const id = ++latest + if (!workspace || paths.length === 0) { + graph = EMPTY_COLUMN_GRAPH + truncated = false + failed = false + loading = false + return + } + // The previous question's failure is not this one's. Cleared as the request + // goes out rather than when it lands, or a retry keeps saying the trace is + // incomplete while it is being fetched, and a new selection inherits the + // last one's failure until its own answer arrives. + failed = false + loading = true + fetchLineage(workspace, paths, pin).then( + (r) => { + if (id !== latest) return + graph = buildDbtColumnGraph(r?.edges ?? []) + truncated = r?.truncated ?? false + failed = false + loading = false + }, + // Lineage annotates a graph that renders without it, so a failed fetch + // leaves that branch unexpanded rather than putting an error over the + // model — but it SAYS so, because an empty trace is what a project + // without the analysis pass looks like, and the two must not read + // alike. Not retried on its own: the effect reruns whenever the canvas + // redraws, and a failing endpoint would then be asked once per redraw. + // Selecting another node and back asks again, and where a caller + // passes `generation`, so does going and fetching the graph. + () => { + if (id !== latest) return + graph = EMPTY_COLUMN_GRAPH + truncated = false + failed = true + loading = false + } + ) + }) + + return { + get graph() { + return graph + }, + get loading() { + return loading + }, + get truncated() { + return truncated + }, + get failed() { + return failed + } + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index de00aef2be..c57687d78b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -319,6 +319,18 @@ describe('resolveGraph', () => { expect(assetTrigKeys(r, 'f/x/open')).toEqual(['ducklake:main.orders']) }) + // A relation a native `// materialize manual dbt://…` script writes wakes its + // subscribers, so hiding the edge would leave the author's own annotation off + // the canvas. The deploy refuses the ones that cannot fire. + it('draws an explicit dbt:// subscription as an unsaved trigger overlay', () => { + const liveAnnotations = { + scriptPath: 'f/x/open', + annotations: ann({ triggerAssets: [{ kind: 'dbt', path: 'main/analytics/orders' }] }) + } + const r = resolveGraph(input({ liveAnnotations })) + expect(assetTrigKeys(r, 'f/x/open')).toEqual(['dbt:main/analytics/orders']) + }) + it('open-script live annotations add unsaved triggers, deduped vs persisted', () => { const base = baseGraph({ triggers: [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index 867fa6f036..5c8aa697a4 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -77,23 +77,15 @@ function persistedNativeKinds(base: AssetGraphResponse, path: string): Set = new Set(['ducklake', 's3object']) -/** Whether a subscription on this kind can ever fire once deployed. - * - * A `dbt://` one cannot: dbt is the only producer of a warehouse relation and - * a dbt run does not dispatch, so the deploy refuses `// on dbt://…` outright - * (`scripts.rs`). The editor must not draw an arrow the deploy will reject — - * applied to the EXPLICIT overlays; auto-derivation is already scoped by - * `AUTO_TRIGGER_KINDS`. Parsing is left alone so the Rust-parity test still - * compares like for like. */ -function canTrigger(kind: AssetKind): boolean { - return kind !== 'dbt' -} - /** `kind:path` refs of a script's `// materialize` write target(s) (base + * the scd2 `_current` companion), which the body `SELECT` doesn't express. */ function materializeWriteRefs(parsed: PipelineAnnotations): string[] { @@ -381,7 +373,7 @@ function makeContext(input: ResolveGraphInput): ResolveContext { const liveRefKeys = new Set() if (openIsSavedEdit) { if (liveAnnotations.scriptPath === openPath) { - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) + for (const a of liveAnnotations.annotations.triggerAssets) liveRefKeys.add(`${a.kind}:${a.path}`) // The `// materialize ` target is a declared *output*, but it // lives in an annotation (not the SQL body), so neither triggerAssets @@ -625,7 +617,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // stable when the user clicks off this draft. Live annotations // (below) take over for the currently-open draft so keystroke // edits still update in real time. - for (const a of parsed.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of parsed.triggerAssets) { extraTriggers.push({ trigger_kind: 'asset', asset_kind: a.kind, @@ -704,7 +696,7 @@ function applyLiveBufferOverlay(acc: Accumulator, input: ResolveGraphInput, ctx: for (let i = extraTriggers.length - 1; i >= 0; i--) { if (extraTriggers[i].runnable_path === livePath) extraTriggers.splice(i, 1) } - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of liveAnnotations.annotations.triggerAssets) { const key = `${a.kind}:${a.path}` if (assetKeys.has(key)) continue extraTriggers.push({ diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 4ada1feec3..023f71e218 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -36,9 +36,14 @@ export interface DbtAssetProvenance { tags?: string[] description?: string data_tests?: DbtDataTest[] - /** Declared column metadata (name -> description). NOT column lineage: - * `manifest.json` carries none (docs/dbt-runtime.md, decision 14). */ + /** Declared column metadata (name -> description): what `manifest.json` + * carries, which is only the columns an author wrote down. */ columns?: Record + /** Every column of the relation, typed and in the order the model produces + * them, from the engine's static analysis. Present only for a project that + * opted into it (`column_lineage: true`); `manifest.json` has no such + * thing. Lockstep with Rust `DbtAssetProvenance.column_schema`. */ + column_schema?: { name: string; type?: string }[] /** A source's declared freshness policy. */ freshness?: unknown /** The model's SQL as written — the transform behind the node. Read-only: diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 23531d71f2..93367a725e 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -96,10 +96,11 @@ export function formatAssetKind(asset: { case 'volume': return 'Volume' case 'dbt': - // The SCHEME says dbt because dbt is the only thing that creates one; - // the PATH stays the relation, so a mart one project builds and the - // `source` the next reads land on one node — their dbt `unique_id`s - // differ where the relation does not (docs/dbt-runtime.md, decision 11). + // The SCHEME says dbt because dbt is what derives these relations; the + // PATH stays the relation, so a mart one project builds, the `source` + // the next reads, and a native `// materialize manual dbt://…` writer + // land on one node — their dbt `unique_id`s differ where the relation + // does not (docs/dbt-runtime.md, decision 11). return 'dbt table' } } diff --git a/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte new file mode 100644 index 0000000000..dadf814a80 --- /dev/null +++ b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte @@ -0,0 +1,323 @@ + + + dispatch('canceled')} + on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })} +> +
+ {#if draftTriggers.length > 0} +
+
+ {`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`} +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + {#each draftTriggers as trigger} + {@const SvelteComponent = triggerIconMap[trigger.type]} + {@const permission = checkSavePermissions(trigger)} + {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} + + + + + + {/each} + + + + + {/if} + + {#if draftAgents.length > 0} +
+
+ Saved agents this flow uses have unsaved changes. Select which ones to deploy with the + flow. An agent kept as a draft stays editable, and the flow runs the agent as currently + deployed. +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + + {#each draftAgents as agent (agent.path)} + {@const permission = checkAgentPermissions(agent)} + {@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)} + + + + + + + + {/each} + + + + + {/if} + + diff --git a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte deleted file mode 100644 index cfebae0e9f..0000000000 --- a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte +++ /dev/null @@ -1,168 +0,0 @@ - - - dispatch('canceled')} - on:confirmed={() => dispatch('confirmed', { selectedTriggers })} -> -
-
- {`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed - draft triggers will be permanently deleted.`} -
- -
5 ? 'h-[300px]' : ''}> - -
- - - - - - - {#each draftTriggers as trigger} - {@const SvelteComponent = triggerIconMap[trigger.type]} - {@const permission = checkSavePermissions(trigger)} - {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} - - - - - - {/each} - - {#if draftTriggers.length === 0} - - - - {/if} - - - - - diff --git a/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts index 4da6f18f32..c7325698e3 100644 --- a/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts +++ b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts @@ -35,9 +35,9 @@ export function useListHighlight(opts: { untrack(() => (index = resting)) }) - function move(delta: number) { + function land(next: number) { const count = opts.count() - if (count === 0) return + if (count === 0 || next < 0 || next >= count) return pointerOwns = false // Rows are tabbable, so focus can sit on one. Enter then activates whatever is // focused, which has to stay the highlighted row — so any row counts, not just @@ -47,16 +47,29 @@ export function useListHighlight(opts: { const focusedId = document.activeElement?.id const rowWasFocused = !!focusedId && Array.from({ length: count }, (_, i) => opts.rowId(i)).includes(focusedId) - index = index < 0 ? (delta > 0 ? 0 : count - 1) : (index + delta + count) % count + index = next const row = document.getElementById(opts.rowId(index)) row?.scrollIntoView({ block: 'nearest' }) if (rowWasFocused) row?.focus() } + function move(delta: number) { + const count = opts.count() + if (count === 0) return + land(index < 0 ? (delta > 0 ? 0 : count - 1) : (index + delta + count) % count) + } + return { get index() { return index }, + /** Step the highlight, for a list whose own keys move it beyond Up and Down — + * a tree stepping into the children a folder just revealed. */ + move, + /** Put the highlight on a row named outright, rather than a step from wherever + * it is — the row a caller's own key landed on, or the one that has focus. A + * step cannot say this: from nothing lit it can only reach an end of the list. */ + moveTo: land, /** Wire to each row's `onMouseEnter`. */ hovered(i: number) { if (pointerOwns) index = i diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte index 674c147c52..bfae0473f1 100644 --- a/frontend/src/lib/components/common/modal/PagedContent.svelte +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -4,7 +4,18 @@ /** One level of a paginated dialog. Order is the order given: the page on screen sits at rest * and every other waits off the side it is listed on, so a deeper page arrives from the right * and the way back arrives from the left without anyone naming a direction. */ - export type ModalPage = { key: string; content: Snippet } + export type ModalPage = { + key: string + content: Snippet + /** + * Drawn in place of `content` for a page that has not been opened yet, so the first + * navigation to it has something to slide in — without one, the box arrives empty and + * fills a frame later, which reads as the animation being broken rather than as + * loading. A skeleton is enough: it is on screen for the length of the transition. + * Unnecessary under `warm`, which builds every page up front. + */ + placeholder?: Snippet + } -{#if href} +{#if allowedHref} {#if wmKind} - + + - {#if previewAction} + + {#if previewAction && !modifier.held} {:else} @@ -110,8 +122,11 @@ {/if} {:else} - + {@render children?.()} {/if} +{:else} + + {@render children?.()} {/if} diff --git a/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte b/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte new file mode 100644 index 0000000000..6072b6e341 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte @@ -0,0 +1,242 @@ + + + + +
+ +
+ +
+
+ {#if hasArgs} + + + {:else} +

This {runnableKind} takes no arguments.

+ {/if} +
+
+ + {#if fades.bottom} +
+ {/if} +
+ + +
+ {#if runForm.clearedKeys?.length} +

+ Sent in a shape this field has no reading of, so it opened empty: + {runForm.clearedKeys.join(', ')} +

+ {/if} + {#if runForm.resetKeys?.length} +

+ Disabled by this {runnableKind}, so it will run with its default: + {runForm.resetKeys.join(', ')} +

+ {/if} + {#if runForm.strippedKeys?.length} +

+ A file, so it opened empty for you to attach: + {runForm.strippedKeys.join(', ')} +

+ {/if} + {#if planMode} +

{PLAN_MODE_MESSAGES.runFormRefused}

+ {/if} + + +
+ + +
+
+
diff --git a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte new file mode 100644 index 0000000000..edd1704bb2 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte @@ -0,0 +1,530 @@ + + + +{#snippet status()} + {#if !pending} + + {statusTime} + + {/if} +{/snippet} + + +{#snippet previewChip()} + +{/snippet} + + + + (toggled = { id: message.tool_call_id, open: !expanded })} + headerLeft={status} + headerRight={previewTarget ? previewChip : undefined} + class="scroll-mb-8" + labelClass="font-main font-medium text-primary" + contentClass="p-0 overflow-hidden" +> + {#if formInPreview} +
+ These inputs are open in the preview panel. +
+ {:else if pending} + + {:else} + +
+ + (userTab = { id: message.tool_call_id, value: e.detail })} + class="h-8 px-3 font-main" + wrapperClass="shrink-0" + slidingIndicator + > + {#if !rawView} + {#each tabs as tab (tab.value)} + + + + {#snippet extra()} + {#if tab.value === 'logs' && logLineCount > 0} + {logLineCount} + {/if} + {/snippet} + + + {/each} + {/if} +
+ (jsonView = { id: message.tool_call_id, on: e.detail })} + size="2xs" + options={{ right: 'JSON', rightTooltip: 'Show this call as raw JSON' }} + lightMode + /> +
+ + + +
+ +
+ {#if rawView} +
+ + + + +
+ {:else} + + {#key activeTab} +
+ {#if activeTab === 'input'} + + + {:else if activeTab === 'logs'} + {#if logs.trim()} + {#if logs.length >= MAX_LOG_LENGTH} +

+ Tail of the logs, the last {MAX_LOG_LENGTH} characters. +

+ {/if} + 
{logs}
+ {:else} +

No logs yet.

+ {/if} + {#if running} +
+ + streaming +
+ {/if} + {:else if failed} + 
{message.error}
+ {:else if streaming} + + + {:else if resultValue !== undefined} + + + {:else if canceled} + +
+ +

{cancelReason}

+ {#if !ran} +

+ The inputs it would have run with are on the Inputs tab. +

+ {/if} +
+ {:else} +

This run returned no result.

+ {/if} +
+ {/key} + {/if} +
+
+ + + {#if fades.bottom && fadeBody} +
+ {#if activeTab === 'logs'} +
+ {/if} + {/if} +
+ + {#if running && chatJob} + +
+ +
+ {/if} + {/if} + diff --git a/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte index c4b2d0cc21..9d1bf698f7 100644 --- a/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte @@ -1,6 +1,7 @@ @@ -136,17 +127,18 @@ {:else if hasContent}
- 
{formatJson($state.snapshot(content))}
- {#if showFade && canScrollDown} + {#if showFade && fades.bottom}
{/if}
diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 34b1adcf68..620357ff87 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -37,8 +37,11 @@ import ToolMessageActions from './ToolMessageActions.svelte' import ToolPreviewCard from './ToolPreviewCard.svelte' import AskUserQuestionDisplay from './AskUserQuestionDisplay.svelte' + import RunScriptCard from './RunScriptCard.svelte' import WebSearchSourcesDisplay from './WebSearchSourcesDisplay.svelte' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' + import McpServerIcon from '$lib/components/mcp/McpServerIcon.svelte' + import { resolveMcpServerMark } from '$lib/components/mcp/serverMark' interface Props { message: ToolDisplayMessage @@ -46,6 +49,11 @@ let { message }: Props = $props() + // Recorded by the call itself, from the connected-server list rather than from the + // model's arguments — which is what lets a reloaded transcript still resolve it, and + // what keeps a path the model made up from being read as a workspace resource. + const mcpServer = $derived(message.mcpServer) + const isPlanReview = $derived(message.toolName === EXIT_PLAN_MODE_TOOL) const isPlanCard = $derived(isPlanCardTool(message.toolName)) const planCopy = $derived( @@ -117,6 +125,10 @@ isActiveUserQuestion(message) ? message.userQuestion : undefined ) + // The run card owns this call from the form to whatever settled it, cancelling included: + // the card is the call, and a run the user stopped is not a different kind of thing. + const isRunCard = $derived(Boolean(message.runForm)) + // The preview chip sits on the header row (to the right of the tool-call text); // shown once the tool settled, never while loading/erroring/awaiting confirmation. const showPreviewChip = $derived( @@ -140,6 +152,8 @@ {message.toolName} {/if} +{:else if isRunCard} + {:else if planState} @@ -235,6 +249,17 @@ {/if} {/snippet} + + {#snippet serverMark()} + {#if mcpServer?.workspace && mcpServer.path} + {#await resolveMcpServerMark(mcpServer.workspace, mcpServer.path) then mark} + + {/await} + {/if} + {/snippet} + @@ -251,6 +276,7 @@ labelClass={showPreviewChip ? 'truncate' : ''} contentClass="space-y-3" headerRight={showPreviewChip ? previewChip : undefined} + headerLeft={mcpServer?.workspace ? serverMark : undefined} > {#snippet belowHeader()} diff --git a/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte b/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte index 81d00fd8e4..b5f4fa8bae 100644 --- a/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte @@ -8,15 +8,26 @@ interface Props { card: { kind: PreviewCardKind; path: string } + /** Opens something other than the item's own preview — the run card opens the call + * it owns, which is a form before it is a run. */ + onOpen?: () => void + title?: string + /** The kind icon says what the chip opens. A card that already names its own runnable + * in the row above has said it, and repeating it there reads as a second subject. */ + kindIcon?: boolean } - let { card }: Props = $props() + let { card, onOpen, title, kindIcon = true }: Props = $props() const kindLabel = $derived(card.kind === 'raw_app' ? 'app' : card.kind) let opening = $state(false) async function open() { if (opening) return + if (onOpen) { + onOpen() + return + } opening = true try { await runToolDisplayAction(openItemPreviewAction(card.kind, card.path)) @@ -30,9 +41,11 @@ variant="default" unifiedSize="2xs" disabled={opening} - title="Open {kindLabel} preview: {card.path}" + title={title ?? `Open ${kindLabel} preview: ${card.path}`} onClick={open} - startIcon={{ icon: RowIcon as unknown as IconType, props: { kind: card.kind, size: 12 } }} + startIcon={kindIcon + ? { icon: RowIcon as unknown as IconType, props: { kind: card.kind, size: 12 } } + : undefined} endIcon={{ icon: PanelRight }} wrapperClasses="shrink-0" > diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte new file mode 100644 index 0000000000..7db5604775 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte @@ -0,0 +1,46 @@ + + + +{#if source} + + {#key content} + + {/key} +{:else} + +
+
+ +
+{/if} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte new file mode 100644 index 0000000000..9fc5f767b4 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte @@ -0,0 +1,42 @@ + + + + diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte new file mode 100644 index 0000000000..4924473e98 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte @@ -0,0 +1,196 @@ + + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+
+ Share with workspace + + {#if share} + Members of {workspace} can open a read-only copy of v{share.version} with this link. + {:else} + Members of {workspace} will be able to open a read-only copy of v{version} with a link. + {/if} + {#if status.current} + The copy is deleted {formatRetention(status.current.retention_secs)} after it is shared. + {/if} + +
+ + {#if share && url} +
+ + + Expires {displayDate(share.expires_at)} + +
+ {#if change} +
+ + {#if change === 'newer'} + The link shows v{share.version}; v{version} is on screen. + {:else if change === 'older'} + The link shows v{share.version}, newer than the v{version} on screen. + {:else} + The link still shows the old name, “{share.name}”. + {/if} + + +
+ {/if} +
+ +
+ {:else} +
+ +
+ {/if} +
+ {/snippet} + diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte index ebc46178d0..8331825ee2 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte @@ -1,23 +1,14 @@
@@ -169,27 +144,22 @@ {/if}
- - + /> + {#if canPreview} {#if restoringPin} - {:else if source} - - {#key `${artifact.id}:${pinnedContent ? `v${pinnedContent.version}` : artifact.updatedAt}`} - - {/key} {:else} - -
-
- -
+ {/if}
diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts new file mode 100644 index 0000000000..115343117a --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/base', () => ({ base: '' })) + +import { shareWorkspaceId } from './artifactSharing' + +describe('shareWorkspaceId', () => { + it('shares a fork session into the topmost workspace the user still belongs to', () => { + const workspaces = [ + { id: 'prod' }, + { id: 'wm-fork-a', parent_workspace_id: 'prod' }, + { id: 'wm-fork-b', parent_workspace_id: 'wm-fork-a' } + ] + expect(shareWorkspaceId('wm-fork-b', workspaces)).toBe('prod') + }) + + it('stops below a parent the user is not a member of', () => { + const workspaces = [{ id: 'wm-fork-a', parent_workspace_id: 'prod' }] + expect(shareWorkspaceId('wm-fork-a', workspaces)).toBe('wm-fork-a') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts new file mode 100644 index 0000000000..57b08cba9b --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts @@ -0,0 +1,42 @@ +import { base } from '$lib/base' + +/** + * The workspace a share from `workspaceId` lands in: the topmost ancestor the user still + * belongs to. A session often runs in a fork, whose members are its creator alone, so a link + * minted there would reach nobody — and it would be deleted with the fork. + */ +export function shareWorkspaceId( + workspaceId: string, + workspaces: { id: string; parent_workspace_id?: string | null }[] +): string { + let current = workspaceId + const seen = new Set([current]) + for (;;) { + const parent = workspaces.find((w) => w.id === current)?.parent_workspace_id + if (!parent || seen.has(parent) || !workspaces.some((w) => w.id === parent)) return current + seen.add(parent) + current = parent + } +} + +export function sharedArtifactUrl(workspaceId: string, shareId: string): string { + return `${window.location.origin}${base}/shared_artifacts/${encodeURIComponent( + shareId + )}?workspace=${encodeURIComponent(workspaceId)}` +} + +/** "30 days", "12 hours": the retention window in the largest whole unit it fills. */ +export function formatRetention(secs: number): string { + const units: [string, number][] = [ + ['day', 86400], + ['hour', 3600], + ['minute', 60] + ] + for (const [unit, size] of units) { + if (secs >= size) { + const n = Math.floor(secs / size) + return `${n} ${unit}${n === 1 ? '' : 's'}` + } + } + return `${secs} second${secs === 1 ? '' : 's'}` +} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts index d70566ad18..a0fb5a4a4b 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts @@ -220,7 +220,7 @@ describe('artifactsDB', () => { expect(await noDb.getArtifact('a1')).toBeUndefined() expect(await noDb.listArtifactsForSession('s1')).toEqual([]) await expect(noDb.deleteArtifact('a1')).resolves.toBeUndefined() - await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBeUndefined() + await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBe(false) }) it('rejects a version read it could not make, instead of reading as absent', async () => { diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts index 94afac404e..f0cb32d71f 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts @@ -2,6 +2,8 @@ // active chat's rotation, so chatId-keying would drop artifacts on each new conversation. import { type DBSchema as IDBSchema, type IDBPObjectStore, type IDBPTransaction } from 'idb' import { userScopedDb } from '$lib/userScopedDb' +import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { markSessionDirty } from '$lib/components/sessions/sessionMirrorSignal' export type ArtifactKind = 'md' | 'html' @@ -89,9 +91,11 @@ interface ArtifactsSchema extends IDBSchema { } } +const ARTIFACTS_DB = 'copilot-artifacts' + // User-scoped like the chat-history store these are keyed against: no cross-user // co-residency on a shared browser. -const dbh = userScopedDb('copilot-artifacts', { +const dbh = userScopedDb(ARTIFACTS_DB, { version: 2, // Runs for a fresh database and for the v1 upgrade alike, so create each store only // when it is missing. @@ -118,11 +122,69 @@ export async function putArtifact(artifact: PersistedArtifact): Promise { // A rejected write (most likely QuotaExceededError) leaves the artifact usable for the // session but unpersisted — degrade like the reads rather than throwing at the caller. await db.put('items', artifact) + markSessionDirty(artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not persist artifact', err) } } +/** A session's artifacts with their history, or undefined when the store is unavailable + * or no longer the named user's (see `readStoredSessions`). */ +export async function readSessionArtifacts( + sessionId: string, + email: string +): Promise<{ items: PersistedArtifact[]; versions: ArtifactVersion[] } | undefined> { + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return undefined + try { + const items = await db.getAllFromIndex('items', 'by-session', sessionId) + const versions = ( + await Promise.all(items.map((i) => db.getAllFromIndex('versions', 'by-artifact', i.id))) + ).flat() + return { items, versions } + } catch (err) { + console.error('Could not read artifacts', err) + return undefined + } +} + +/** Write restored artifacts and snapshots, leaving any that already exist alone. Reports + * whether they are all in the store now: unlike the other writes here, a caller records the + * restore as done on the strength of this answer. */ +export async function importArtifacts( + items: PersistedArtifact[], + versions: ArtifactVersion[], + email: string, + overwrite = false +): Promise { + if (items.length === 0 && versions.length === 0) return true + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const itemStore = tx.objectStore('items') + const versionStore = tx.objectStore('versions') + // An overwrite never puts an older record over a newer one: without a cross-tab lock, + // another restore may have landed a newer backup's copy meanwhile. + for (const item of items) { + const existing = await itemStore.get(item.id) + if (existing === undefined || (overwrite && existing.updatedAt <= item.updatedAt)) { + await itemStore.put(item) + } + } + for (const version of versions) { + if (overwrite || (await versionStore.getKey(version.key)) === undefined) { + await versionStore.put(version) + } + } + await tx.done + return true + } catch (err) { + console.error('Could not import artifacts', err) + return false + } +} + export async function getArtifact(id: string): Promise { const db = await getDB() if (!db) return undefined @@ -273,7 +335,11 @@ export async function mutateArtifact( reportFailure = false abort() } - return { outcome: await settled, artifact: edit.artifact } + const outcome = await settled + if (outcome === 'saved' && db) { + markSessionDirty(edit.artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) + } + return { outcome, artifact: edit.artifact } } /** @@ -342,17 +408,55 @@ export async function deleteArtifact(id: string): Promise { if (!db) return try { const tx = db.transaction(['items', 'versions'], 'readwrite') - await tx.objectStore('items').delete(id) + const items = tx.objectStore('items') + const sessionId = (await items.get(id))?.sessionId + await items.delete(id) await deleteVersionsIn(tx.objectStore('versions'), id) await tx.done + if (sessionId) markSessionDirty(sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not delete artifact', err) } } -export async function deleteArtifactsForSession(sessionId: string): Promise { +/** Deletes these artifacts of the session (with their versions) and these versions: what + * an earlier restore staged for it and the backup no longer has. False when nothing could + * be deleted. */ +export async function pruneSessionArtifacts( + sessionId: string, + itemIds: Set, + versionKeys: Set, + email: string +): Promise { + if (itemIds.size === 0 && versionKeys.size === 0) return true const db = await getDB() - if (!db) return + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const items = tx.objectStore('items') + const versions = tx.objectStore('versions') + for (const id of await items.index('by-session').getAllKeys(sessionId)) { + if (!itemIds.has(String(id))) continue + await items.delete(id) + await deleteVersionsIn(versions, String(id)) + } + for (const key of versionKeys) await versions.delete(key) + await tx.done + return true + } catch (err) { + console.error('Could not prune artifacts for session', err) + return false + } +} + +/** False when the store could not be reached or the deletion failed. With `email`, only that + * user's store is touched: a caller that captured its user must not follow an account switch. */ +export async function deleteArtifactsForSession( + sessionId: string, + email?: string +): Promise { + const db = await getDB() + if (!db || (email !== undefined && db.name !== scopedKeyFor(ARTIFACTS_DB, email))) return false try { const tx = db.transaction(['items', 'versions'], 'readwrite') const items = tx.objectStore('items') @@ -365,8 +469,10 @@ export async function deleteArtifactsForSession(sessionId: string): Promise string[] +export type PathsPreference = { + /** Paths stored as on, which is the enabled set only for a kind that defaults to + * off. For one that defaults to on this is whatever happens to be written down — + * nothing for a decision made since the default flipped, and the whole selection + * for a browser still carrying storage from before it. Neither is the answer to + * "what is enabled": ask `isEnabled` per path for that. */ + explicitlyEnabledPaths: (workspace: string) => string[] isEnabled: (workspace: string, path: string) => boolean /** Returns false when there is no account to record the preference against, so * a caller that just created the object can say it did not stay on. */ setEnabled: (workspace: string, path: string, enabled: boolean) => boolean + /** Drop any decision about `path`, leaving it at the default. What a caller wants + * when the thing at that path is gone — a deleted or renamed skill — rather than + * writing the default as a decision, which reads as one and would mean the + * opposite the day the default moves. */ + forget: (workspace: string, path: string) => void } -export function createEnabledPathsPreference(storageKey: string): EnabledPathsPreference { +/** One decision per path. The older shape was an array of the paths that were on, + * which says exactly that and so needs no conversion to be read. */ +type Choices = Record + +function toChoices(raw: unknown): Choices { + if (Array.isArray(raw)) return Object.fromEntries(raw.map((path) => [String(path), true])) + if (raw && typeof raw === 'object') { + return Object.fromEntries( + Object.entries(raw as Record).map(([path, on]) => [path, on === true]) + ) + } + return {} +} + +export function createPathsPreference( + storageKey: string, + defaultEnabled: boolean +): PathsPreference { function scope(workspace: string): string | undefined { const email = get(userStore)?.email return email ? `${workspace}:${email}` : undefined } - function read(): Record { + function read(): Record { if (typeof localStorage === 'undefined') return {} try { return JSON.parse(localStorage.getItem(storageKey) ?? '{}') @@ -40,35 +65,42 @@ export function createEnabledPathsPreference(storageKey: string): EnabledPathsPr } } - function write(all: Record) { + function choices(workspace: string): Choices { + const key = scope(workspace) + return key ? toChoices(read()[key]) : {} + } + + /** `decision` of undefined drops the entry. */ + function write(workspace: string, path: string, decision: boolean | undefined): boolean { + const key = scope(workspace) + if (!key) return false + const all = read() + const current = toChoices(all[key]) + if (decision === undefined) { + delete current[path] + } else { + current[path] = decision + } + all[key] = current try { localStorage.setItem(storageKey, JSON.stringify(all)) } catch (e) { console.error(`Failed to persist ${storageKey}`, e) } - } - - function enabledPaths(workspace: string): string[] { - const key = scope(workspace) - return key ? (read()[key] ?? []) : [] + return true } return { - enabledPaths, - isEnabled: (workspace, path) => enabledPaths(workspace).includes(path), - setEnabled: (workspace, path, enabled) => { - const key = scope(workspace) - if (!key) return false - const all = read() - const current = new Set(all[key] ?? []) - if (enabled) { - current.add(path) - } else { - current.delete(path) - } - all[key] = [...current] - write(all) - return true - } + explicitlyEnabledPaths: (workspace) => + Object.entries(choices(workspace)) + .filter(([, on]) => on) + .map(([path]) => path), + isEnabled: (workspace, path) => choices(workspace)[path] ?? defaultEnabled, + // A path put back to the default is dropped rather than stored as one: the + // entries are the decisions, so a re-enabled skill leaves nothing behind and a + // later skill at that path starts from the default like any other. + setEnabled: (workspace, path, enabled) => + write(workspace, path, enabled === defaultEnabled ? undefined : enabled), + forget: (workspace, path) => void write(workspace, path, undefined) } } diff --git a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts index 9673e29edc..59d5d57204 100644 --- a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts +++ b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts @@ -19,7 +19,7 @@ describe('attachedFilesDB without IndexedDB', () => { putItem({ id: 'a', sessionId: 's1', kind: 'snapshot', name: 'x.txt', addedAt: 0 }) ).resolves.toBeUndefined() await expect(deleteItem('a')).resolves.toBeUndefined() - await expect(deleteItemsForSession('s1')).resolves.toBeUndefined() + await deleteItemsForSession('s1') }) it('does not throw when requesting persistent storage', async () => { diff --git a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts index fe051ef8cc..dc872845cd 100644 --- a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts +++ b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts @@ -91,9 +91,10 @@ export async function deleteItem(id: string): Promise { await db?.delete('items', id) } -export async function deleteItemsForSession(sessionId: string): Promise { +/** False when the store could not be reached or the deletion failed. */ +export async function deleteItemsForSession(sessionId: string): Promise { const db = await getDB() - if (!db) return + if (!db) return false try { const tx = db.transaction('items', 'readwrite') const index = tx.store.index('by-session') @@ -103,8 +104,10 @@ export async function deleteItemsForSession(sessionId: string): Promise { cursor = await cursor.continue() } await tx.done + return true } catch (err) { console.error('Could not delete attached files for session', err) + return false } } diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 47a6b0b169..c6bed6988c 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -14,6 +14,9 @@ import type { ScriptLintResult } from '../shared' import { applyFlowJsonUpdate, updateRawScriptModuleContent } from './helperUtils' import { findModuleInFlow } from '$lib/components/flows/flowTree' + import { getEditorStoragePath } from '$lib/components/editorStoragePathContext' + + const editorStoragePath = getEditorStoragePath() let { flowModuleSchemaMap, @@ -162,6 +165,8 @@ selectionManager.selectId(id, { openPanel: true }) }, + getStoragePath: () => editorStoragePath?.(), + testFlow: async (args, conversationId) => { // Set preview args if provided if (args) { diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 63c4ffe94f..51fea34d58 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -136,6 +136,10 @@ export interface FlowAIChatHelpers { /** Run a test of the current flow using the UI's preview mechanism */ testFlow: (args?: Record, conversationId?: string) => Promise + /** The path this editor's draft is stored under. Tells a caller which of several mounted + * editors is the one an active-editor context names. */ + getStoragePath: () => string | undefined + /** Get lint errors from a specific module (focuses it first, waits for Monaco to analyze) */ getLintErrors: (moduleId: string) => Promise } diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlow.json b/frontend/src/lib/components/copilot/chat/flow/openFlow.json index cab2839d48..03b839dd98 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlow.json +++ b/frontend/src/lib/components/copilot/chat/flow/openFlow.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"version":"1.791.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file +{"openapi":"3.0.3","info":{"version":"1.811.1","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments/enabled_tools).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts index 5bd36b85ce..667a597b57 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts @@ -1,6 +1,6 @@ import { z } from "zod" -export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "enabled_tools": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -17,10 +17,10 @@ export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "i message: "Invalid input: Should pass single schema", }); } - }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type") + }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments/enabled_tools).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type") -export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "enabled_tools": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -37,7 +37,7 @@ export const flowModuleSchema = z.object({ "id": z.string().describe("Unique ide message: "Invalid input: Should pass single schema", }); } - }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") + }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments/enabled_tools).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") export const flowModulesSchema = z.array(flowModuleSchema) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts index 8158fe62b8..0b1ea0952e 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts @@ -88,6 +88,32 @@ const CATALOG = [ required: ['workspace', 'hash'] } }, + { + name: 'runScriptByPath', + description: 'Run script by path', + instructions: 'Trigger a run of a deployed script', + path: '/w/{workspace}/jobs/run/p/{path}', + method: 'POST', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, path: { type: 'string' } }, + required: ['workspace', 'path'] + }, + body_schema: { type: 'object', additionalProperties: true } + }, + { + name: 'cancelQueuedJob', + description: 'Cancel a queued job', + instructions: '', + path: '/w/{workspace}/jobs_u/queue/cancel/{id}', + method: 'POST', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, id: { type: 'string' } }, + required: ['workspace', 'id'] + }, + body_schema: { type: 'object', additionalProperties: true } + }, { name: 'runFlowByPath', description: 'Run flow by path', @@ -166,7 +192,7 @@ describe('call_api_get', () => { const unknown = await run('call_api_get', { name: 'nope' }) expect(unknown.error).toContain('search_api_endpoints') - const mutating = await run('call_api_get', { name: 'runFlowByPath' }) + const mutating = await run('call_api_get', { name: 'cancelQueuedJob' }) expect(mutating.error).toContain('call_api_endpoint') const deleting = await run('call_api_endpoint', { name: 'deleteSchedule' }) @@ -178,6 +204,24 @@ describe('call_api_get', () => { expect(search.matches.map((m: any) => m.name)).not.toContain('deleteScriptByHash') }) + // Left reachable, these endpoints are the way around the argument form: they run the + // deployed runnable on the model's arguments, unstripped and unshown. + it('refuses a deployed run, pointing at run_script and run_flow', async () => { + for (const [name, tool, query] of [ + ['runScriptByPath', 'run_script', 'run deployed script'], + ['runFlowByPath', 'run_flow', 'run deployed flow'] + ]) { + const called = await run('call_api_endpoint', { name }) + expect(called.error).toContain(tool) + expect(called.success).toBe(false) + + // And it is gone from search, so the model is redirected before it ever calls. + const search = await run('search_api_endpoints', { query }) + expect(search.matches.map((m: any) => m.name)).not.toContain(name) + expect(search.covered_by_dedicated_tools?.join(' ')).toContain(tool) + } + }) + it('refuses draft-blind item reads and lists, pointing at the draft-aware tools', async () => { for (const name of ['getScriptByPath', 'getResource', 'getSchedule']) { const result = await run('call_api_get', { name }) @@ -233,11 +277,11 @@ describe('call_api_endpoint', () => { }) vi.stubGlobal('fetch', fetchMock) const result = await run('call_api_endpoint', { - name: 'runFlowByPath', - params: { path: 'u/me/myflow' }, + name: 'cancelQueuedJob', + params: { id: 'job/1' }, body: { args: { n: 1 } } }) - expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs/run/f/u%2Fme%2Fmyflow', { + expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs_u/queue/cancel/job%2F1', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ args: { n: 1 } }) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts index 5d1324ebe3..42d45916eb 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts @@ -36,6 +36,8 @@ const COVERED_ENDPOINTS: Record = { listFlows: 'list_workspace_items (it includes your drafts)', listResource: 'list_workspace_items (it includes your drafts)', listSchedules: 'list_workspace_items (it includes your drafts)', + runScriptByPath: 'run_script (it shows the user an argument form to confirm)', + runFlowByPath: 'run_flow (it shows the user an argument form to confirm)', deleteScriptByPath: 'delete_workspace_item', deleteScriptByHash: 'delete_workspace_item', deleteFlowByPath: 'delete_workspace_item', diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 19050bb7da..051304cafa 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -82,12 +82,21 @@ vi.mock('$lib/gen', async () => { runScriptPreview: vi.fn(async () => 'job-script-preview'), runFlowPreview: vi.fn(async () => 'job-flow-preview'), runFlowByPath: vi.fn(async () => 'job-flow-by-path'), + runScriptByPath: vi.fn(async () => 'job-script-by-path'), getJob: vi.fn(async () => ({ type: 'CompletedJob', success: true, result: { ok: true }, logs: 'test logs' })), + // What every job wait polls first; unmocked it reaches the real client and the + // wait never returns. Answers completed, so one tick settles the job. + getJobUpdates: vi.fn(async () => ({ + completed: true, + running: false, + new_logs: 'test logs', + log_offset: 'test logs'.length + })), getJobLogs: vi.fn(async () => 'job log line 1\njob log line 2'), listJobs: vi.fn(async () => [ { @@ -211,6 +220,9 @@ vi.mock('$lib/gen', async () => { createResource: vi.fn(async () => 'created'), updateResource: vi.fn(async () => 'updated'), deleteResource: vi.fn(async () => 'deleted'), + // A workspace with no skills, which is what makes `read_skill` refuse a path + // the model composed: the gate is membership in the listing. + listResource: vi.fn(async () => []), getResourceValue: vi.fn(async () => ({ content: 'skill body' })) }), VariableService: wrapService(actual.VariableService, { @@ -294,6 +306,12 @@ vi.mock('$lib/gen', async () => { } }) +// Minting reaches the API and is covered in secretArgUtils.test.ts; what matters here is that a +// run the posture answers goes through it and starts on what came back. +vi.mock('$lib/components/secretArgUtils', () => ({ + processSecretArgs: vi.fn(async (args: Record) => args) +})) + vi.mock('./rawAppBundlerBridge', () => ({ bundleRawAppDraft: vi.fn(async () => ({ js: 'bundled js', @@ -303,11 +321,13 @@ vi.mock('./rawAppBundlerBridge', () => ({ vi.mock('$lib/infer', async () => ({ ...(await vi.importActual('$lib/infer')), - // Avoid the wasm parser in unit tests: the script deploy path infers the arg - // schema but tolerates failure, and these tests don't assert on the schema. + // Avoid the wasm parser in unit tests. A no-op passes the seeded schema through + // untouched, which is what lets the password-marking test pin how a schema is + // seeded in and carried out without pinning inference's own merge rules. inferArgs: vi.fn(async () => {}) })) +import { inferArgs } from '$lib/infer' import { buildRunsFilterSearchbarSchema } from '$lib/components/runs/runsFilter' import { buildOpenPageUrl, @@ -349,6 +369,7 @@ import { VariableService } from '$lib/gen' import { superadmin, userStore, usersWorkspaceStore } from '$lib/stores' +import { processSecretArgs } from '$lib/components/secretArgUtils' import { clearWorkspaceRoleCache } from '$lib/user' import { get } from 'svelte/store' import type { Tool, ToolCallbacks } from '../shared' @@ -366,9 +387,22 @@ function getBackendDraft(kind: string, path: string, _opts?: unknown): return backendDrafts.get(`${kind}:${path}`) as V | undefined } +// inferArgs is stubbed module-wide (no wasm parser here), so a test whose form is built by +// inference has to say what the next call finds. Once, so a test that also calls write_script +// — which infers to fill the draft's schema — queues this after that write, not before. +function stubInferredProperties(properties: Record): void { + vi.mocked(inferArgs).mockImplementationOnce(async (_lang, _code, schema) => { + schema.properties = properties + return null + }) +} + const toolCallbacks: ToolCallbacks = { setToolStatus: vi.fn(), - removeToolStatus: vi.fn() + removeToolStatus: vi.fn(), + // Every host that can run a script mounts the form, so the default answers it with what it + // opened with. A test meaning to exercise a host without one overrides this with undefined. + requestRunArgs: async (_toolId, form) => form.args } function getGlobalTool(name: string): Tool<{}> { @@ -4318,8 +4352,7 @@ describe('global AI tools', () => { const result = await withCompletedTestJob(() => callGlobalTool('test_run_script', { - path: 'f/scripts/draft-test', - args: { name: 'Ada' } + path: 'f/scripts/draft-test' }) ) @@ -4328,7 +4361,7 @@ describe('global AI tools', () => { requestBody: { path: 'f/scripts/draft-test', content, - args: { name: 'Ada' }, + args: {}, language: 'bun' } }) @@ -4337,6 +4370,43 @@ describe('global AI tools', () => { expect(result).toContain('test logs') }) + // No parser emits `password`, so the stored schema is the only thing carrying it: an edit + // that rewrites the draft's schema from scratch, or a draft read that drops it, unmarks + // the field — and the form then takes the secret as a plain literal into the job's args. + it('test_run_script keeps the password marking of the script it previews', async () => { + vi.mocked(ScriptService.existsScriptByPath).mockResolvedValueOnce(true) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/secretful', + language: 'bun', + schema: { + type: 'object', + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + } as any) + + await callGlobalTool('write_script', { + path: 'f/scripts/secretful', + language: 'bun', + content: 'export async function main(token: string) { return 1 }' + }) + + let form: any + await callGlobalTool( + 'test_run_script', + { path: 'f/scripts/secretful' }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, opened) => { + form = opened + return undefined + } + } + ) + + expect(form?.schema?.properties).toMatchObject({ token: { password: true } }) + }) + it('test_run_script previews deployed script content when no draft exists', async () => { vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ path: 'f/scripts/deployed-test', @@ -4347,8 +4417,7 @@ describe('global AI tools', () => { await withCompletedTestJob(() => callGlobalTool('test_run_script', { - path: 'f/scripts/deployed-test', - args: { name: 'Grace' } + path: 'f/scripts/deployed-test' }) ) @@ -4361,18 +4430,412 @@ describe('global AI tools', () => { requestBody: { path: 'f/scripts/deployed-test', content: 'def main(name):\n return name', - args: { name: 'Grace' }, + args: {}, language: 'python3' } }) }) + // A test run meets the same card as a deployed one, so what previews is what the form + // submitted — not what the model proposed. + it('test_run_script previews the arguments the form submitted', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/formed-test', + summary: 'Formed test script', + content: 'export async function main(name: string) {}', + language: 'bun', + schema: { properties: { name: { type: 'string' } } } + } as any) + + let opened: Record | undefined + let kind: string | undefined + let helperSource: { code?: string; lang?: string } | undefined + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_script', + { path: 'f/scripts/formed-test', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + opened = form.args + kind = form.kind + helperSource = { code: form.code, lang: form.lang } + return { name: 'Grace' } + } + } + ) + ) + + expect(opened).toEqual({ name: 'Ada' }) + // Drives the card's tense: a test says it tested, not that it ran. + expect(kind).toBe('test') + // The draft itself, so a dynselect field offers the options this code returns rather + // than the deployed version's — which is stale, or absent for a draft never deployed. + expect(helperSource).toEqual({ + code: 'export async function main(name: string) {}', + lang: 'bun' + }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/scripts/formed-test', + content: 'export async function main(name: string) {}', + args: { name: 'Grace' }, + language: 'bun' + } + }) + }) + + // The bypass posture answers a run form as it answers any other confirmation, for a + // deployed run as much as a test. The card must never render one first: a form nobody + // will fill in is attached already settled, so no field is ever mounted, and the schema + // it would have built them from never reaches the transcript. + it('mounts no field on either run form under yolo', async () => { + const script = { + path: 'f/scripts/yolo', + content: 'export async function main(name: string) {}', + language: 'bun', + schema: { properties: { name: { type: 'string' } } } + } as any + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) + + const statuses: any[] = [] + const requestRunArgs = vi.fn(async (_toolId: string, form: any) => form.args) + const yolo = { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs + } + + await withCompletedTestJob(() => + callGlobalTool('test_run_script', { path: 'f/scripts/yolo', args: { name: 'Ada' } }, yolo) + ) + + const testForm = statuses.find((s) => s.runForm)?.runForm + expect(testForm.submitted).toBe(true) + // Nothing is left to render it, and a card carrying one persists it forever. + expect(testForm.schema).toBeUndefined() + // Told the form is already answered, or the loop parks on a card with no fields. + expect(requestRunArgs.mock.calls[0][2]).toEqual({ autoAccepted: true }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: expect.objectContaining({ args: { name: 'Ada' } }) }) + ) + + statuses.length = 0 + await withCompletedTestJob(() => + callGlobalTool('run_script', { path: 'f/scripts/yolo', args: { name: 'Ada' } }, yolo) + ) + + const deployedForm = statuses.find((s) => s.runForm)?.runForm + expect(deployedForm.submitted).toBe(true) + expect(deployedForm.schema).toBeUndefined() + // skipPreprocessor: the form fills the main input schema, so a preprocessor would take + // these arguments for a webhook body and run main on its output instead. + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' }, skipPreprocessor: true }) + ) + }) + + // The posture is the user's standing answer to whether to ask, so a run it answers starts on + // the model's arguments as sent — no default filled in, no required field second-guessed. + // Predicting what a mounted field would hold starts runs the form itself would refuse; the + // schema's own defaults are the worker's job, from the code's signature. + it('sends the model arguments as proposed when the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/defaulted', + schema: { + properties: { name: { type: 'string' }, retries: { type: 'number', default: 3 } }, + required: ['name', 'retries'] + } + } as any) + const statuses: any[] = [] + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/defaulted', args: { name: 'Ada' } }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(statuses.find((x) => x.runForm)?.runForm.submitted).toBe(true) + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' } }) + ) + }) + + // A disabled field is declared as not the caller's to set, and the posture answering the + // form does not make the model one of the callers it is kept from. + it('holds a disabled default against the model when the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/locked', + schema: { + properties: { + name: { type: 'string' }, + mode: { type: 'string', disabled: true, default: 'safe' } + } + } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/locked', args: { name: 'Ada', mode: 'destructive' } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada', mode: 'safe' } }) + ) + // Or the next call proposes the same override again. + expect(result).toContain('disables mode') + }) + + // With no form there is no PasswordArgInput to turn a proposed secret into a reference, and + // a job's arguments are readable by everyone who can see its run. What starts the job must + // be what came back from the minting, never the proposal. + it('mints a proposed secret into a reference before starting a run the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/secret', + schema: { properties: { token: { type: 'string', password: true } }, required: ['token'] } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/secret', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(processSecretArgs).toHaveBeenCalledWith( + { token: 'hunter2' }, + expect.anything(), + expect.anything() + ) + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The bypass is the user's standing answer, not a licence for the host to skip asking: + // a chat with nowhere to put a form still refuses the run under any other posture. + it('run_script refuses a host with no form unless the posture answers for it', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noform', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const refused = await callGlobalTool( + 'run_script', + { path: 'f/scripts/noform', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: undefined } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(refused).toContain('cannot show a run form') + }) + + // The posture answers wherever it is set, form or no form: what it answers is consent, and a + // host without one has nothing left to ask. A secret still becomes a reference first, which + // is the only thing the missing form would have done. + it('run_script runs on a formless host under yolo', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noform-secret', + schema: { + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/noform-secret', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + requestRunArgs: undefined, + shouldAutoAcceptToolConfirmations: () => true + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The transcript is re-cloned into IndexedDB on every save, and a form takes as much text + // as the user pastes. What the card stores is bounded; what the job runs is not. + it('run_script stores a marker for oversized arguments but runs them in full', async () => { + const huge = 'x'.repeat(120_000) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/big', + content: 'export async function main(blob: string) {}', + language: 'bun', + schema: { properties: { blob: { type: 'string' } } } + } as any) + + const statuses: any[] = [] + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/big', args: { blob: 'small' } }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + // The user pastes into the field the model left small: the model's own + // proposal is bounded by what it can emit, this is not. + requestRunArgs: async () => ({ blob: huge }) + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { blob: huge } }) + ) + const persisted = statuses.filter((s) => s.parameters !== undefined).at(-1)?.parameters + expect(persisted).toEqual({ reason: 'WINDMILL_TOO_BIG' }) + expect(JSON.stringify(statuses)).not.toContain(huge) + }) + + // A dropped pass-through fails silently: the argument is accepted and ignored, and the + // run just waits out the default budget. + it('run_script detaches on background and on wait_seconds', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/slow', + schema: { properties: { name: { type: 'string' } } } + } as any) + + // wait_seconds 0 detaches the same way, so one run of each pins both lines. + for (const detachArg of [{ background: true }, { wait_seconds: 0 }]) { + const onJobDetached = vi.fn() + await callGlobalTool( + 'run_script', + { path: 'f/scripts/slow', args: { name: 'Ada' }, ...detachArg }, + { ...toolCallbacks, onJobStarted: vi.fn(), onJobDetached } + ) + + expect(onJobDetached).toHaveBeenCalledWith('job-script-by-path') + } + }) + + // A schema with no fields still opens a form: an empty one is still the Run button, and + // that button is the whole confirmation this tool has. Skipping it because there is + // nothing to fill in starts the script with no confirmation at all. + it('test_run_script opens a form and starts no job when the schema declares no field', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noargs-test', + content: 'export async function main() {}', + language: 'bun', + schema: { properties: {} } + } as any) + + const cancelled = await callGlobalTool( + 'test_run_script', + { path: 'f/scripts/noargs-test', args: { force_delete: true } }, + { ...toolCallbacks, requestRunArgs: async () => undefined } + ) + + expect(JobService.runScriptPreview).not.toHaveBeenCalled() + expect(cancelled).toContain('The user cancelled the run form') + + // A schema declaring nothing is a form with no field to hold this, and the card says + // as much — so answering it must not send an argument that was never on screen. + const answered = await withCompletedTestJob(() => + callGlobalTool( + 'test_run_script', + { path: 'f/scripts/noargs-test', args: { force_delete: true } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(JobService.runScriptPreview).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: expect.objectContaining({ args: {} }) }) + ) + expect(answered).toContain('does not declare force_delete') + }) + + // A host that answers the form without minting, as the eval harness does by returning the + // proposal verbatim: the job's arguments are readable by everyone who can see its run. + it('mints a secret the host handed back as a literal', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/host-literal', + schema: { properties: { token: { type: 'string', password: true } } } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/host-literal', args: { token: 'hunter2' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The bypass has no form to have shown them either, so the rule holds there too. + it('drops an undeclared argument when the posture answers too', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/noargs-yolo', + schema: { properties: { name: { type: 'string' } } } + } as any) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/noargs-yolo', args: { name: 'Ada', force_delete: true } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' } }) + ) + }) + + // The form offers the arguments the flow declares and no others, so a fixture flow that + // takes one has to say so — as a real flow does, since nothing else could render a field. + const FLOW_NAME_SCHEMA = { type: 'object', properties: { name: { type: 'string' } } } + it('test_run_flow previews draft flow content by path', async () => { const modules = [{ id: 'start', value: { type: 'identity' } }] await callGlobalTool('write_flow', { path: 'f/flows/draft-test', summary: 'Draft test flow', - modules: JSON.stringify(modules) + modules: JSON.stringify(modules), + schema: JSON.stringify(FLOW_NAME_SCHEMA) }) await withCompletedTestJob(() => @@ -4399,7 +4862,7 @@ describe('global AI tools', () => { path: 'f/flows/deployed-test', summary: 'Deployed test flow', value: { modules }, - schema: {} + schema: FLOW_NAME_SCHEMA } as any) await withCompletedTestJob(() => @@ -4423,15 +4886,17 @@ describe('global AI tools', () => { }) }) - it('test_run_flow uses the live flow editor test hook when the active editor matches the path', async () => { + // The editor is driven by the key its draft is stored under, which reads and edits of the + // path resolve through too: a staged rename leaves that key where it was. + it('test_run_flow drives the live flow editor by its storage path', async () => { seedBackendDraft( 'flow', - '', + 'u/admin/live_flow_storage', { path: 'u/admin/live_flow', summary: 'Live flow', value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, @@ -4442,7 +4907,7 @@ describe('global AI tools', () => { UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', - storagePath: '', + storagePath: 'u/admin/live_flow_storage', effectivePath: 'u/admin/live_flow' }) const testActiveFlow = vi.fn(async () => 'job-live-flow') @@ -4454,12 +4919,14 @@ describe('global AI tools', () => { path: 'u/admin/live_flow', args: { name: 'Ada' } }, - toolCallbacks, + { ...toolCallbacks, requestRunArgs: async () => ({ name: 'Grace' }) }, { testActiveFlow } ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + // What the form submitted, not what the model proposed: the editor runs the flow, but + // the arguments are the user's. + expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_storage', { name: 'Grace' }) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).not.toHaveBeenCalled() expect(result).toContain('Result (SUCCESS)') @@ -4468,12 +4935,12 @@ describe('global AI tools', () => { it('test_run_flow falls back to preview when the live flow editor test hook returns undefined', async () => { seedBackendDraft( 'flow', - '', + 'u/admin/live_flow_fallback', { path: 'u/admin/live_flow_fallback', summary: 'Live flow fallback', value: { modules: [{ id: 'fallback_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, @@ -4484,7 +4951,7 @@ describe('global AI tools', () => { UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', - storagePath: '', + storagePath: 'u/admin/live_flow_fallback', effectivePath: 'u/admin/live_flow_fallback' }) const testActiveFlow = vi.fn(async () => undefined) @@ -4501,7 +4968,7 @@ describe('global AI tools', () => { ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_fallback', { name: 'Ada' }) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).toHaveBeenCalledWith({ workspace: WORKSPACE, @@ -4513,8 +4980,223 @@ describe('global AI tools', () => { }) }) + // A flow reaches its run form the way a script does: opened on the flow's own input schema, + // with the dynamic-option picker the schema carries, and the run takes what came back. + it('test_run_flow opens the form on the flow schema and runs what it submitted', async () => { + const modules = [{ id: 'start', value: { type: 'identity' } }] + await callGlobalTool('write_flow', { + path: 'f/flows/formed-flow', + summary: 'Formed flow', + modules: JSON.stringify(modules), + schema: JSON.stringify({ + ...FLOW_NAME_SCHEMA, + 'x-windmill-dyn-select-code': 'export function names() { return ["Ada"] }', + 'x-windmill-dyn-select-lang': 'bun' + }) + }) + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'f/flows/formed-flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + } + ) + ) + + expect(form.args).toEqual({ name: 'Ada' }) + expect(form.runnableKind).toBe('flow') + expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) + // The flow's own dynselect script, which the schema carries rather than a step. + expect({ code: form.code, lang: form.lang }).toEqual({ + code: 'export function names() { return ["Ada"] }', + lang: 'bun' + }) + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/flows/formed-flow', + value: { modules }, + args: { name: 'Grace' } + } + }) + }) + + // The form waits as long as the user does, so which editor is on screen is only known when + // they press Run — checking it when the card appeared would drive an editor they have since + // moved away from. + it('test_run_flow re-checks the editor on screen when the form is submitted', async () => { + const modules = [{ id: 'moved_step', value: { type: 'identity' } }] + seedBackendDraft( + 'flow', + 'u/admin/moved_flow', + { + path: 'u/admin/moved_flow', + summary: 'Moved flow', + value: { modules }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/moved_flow', + effectivePath: 'u/admin/moved_flow' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/moved_flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + // The preview panel moves to another flow while the form sits open. + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/other_flow', + effectivePath: 'u/admin/other_flow' + }) + return form.args + } + }, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { path: 'u/admin/moved_flow', value: { modules }, args: { name: 'Ada' } } + }) + }) + + // The flow may be open in a session tab that isn't the one on screen: driving its editor + // would paint the run into a tab the user is not looking at. + it('test_run_flow previews rather than driving an editor the user is not looking at', async () => { + seedBackendDraft( + 'flow', + 'u/admin/background_flow', + { + path: 'u/admin/background_flow', + summary: 'Background flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/flow_on_screen', + effectivePath: 'u/admin/flow_on_screen' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/background_flow', args: { name: 'Ada' } }, + toolCallbacks, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'u/admin/background_flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + args: { name: 'Ada' } + } + }) + }) + + // What separates a deployed run from a test run of the same flow: the form is built from the + // deployed schema rather than the draft's, and the editor open on that path is left alone — + // it holds the draft, so a deployed run painted into its graph would show steps that are not + // the ones running. + it('run_flow forms on the deployed flow and leaves the live editor alone', async () => { + seedBackendDraft( + 'flow', + 'u/admin/deployed_and_drafted', + { + path: 'u/admin/deployed_and_drafted', + summary: 'Draft of the deployed flow', + value: { modules: [{ id: 'draft_step', value: { type: 'identity' } }] }, + schema: { type: 'object', properties: { draft_only: { type: 'string' } } }, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/deployed_and_drafted', + effectivePath: 'u/admin/deployed_and_drafted' + }) + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + path: 'u/admin/deployed_and_drafted', + summary: 'Deployed flow', + value: { modules: [{ id: 'deployed_step', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA + } as any) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'run_flow', + { path: 'u/admin/deployed_and_drafted', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + }, + { testActiveFlow } + ) + ) + + expect(form.runnableKind).toBe('flow') + expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(JobService.runFlowByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'u/admin/deployed_and_drafted', + requestBody: { name: 'Grace' }, + skipPreprocessor: true + }) + }) + it('test_run_step previews rawscript steps from the draft flow', async () => { const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' + // The form offers the fields the step's own code declares, so the step needs a schema + // for `name` to survive it. + stubInferredProperties({ name: { type: 'string' } }) await callGlobalTool('write_flow', { path: 'f/flows/rawscript-step', summary: 'Flow with rawscript', @@ -4572,6 +5254,9 @@ describe('global AI tools', () => { ]) }) + // A script draft carries no schema, so the form infers from the draft content — the + // version about to run. + stubInferredProperties({ name: { type: 'string' } }) await withCompletedTestJob(() => callGlobalTool('test_run_step', { path: 'f/flows/script-step', @@ -4597,7 +5282,10 @@ describe('global AI tools', () => { await callGlobalTool('write_flow', { path: 'f/flows/nested-draft', summary: 'Nested draft flow', - modules: JSON.stringify(nestedModules) + modules: JSON.stringify(nestedModules), + // A subflow step's form is the subflow's own inputs, so `name` needs declaring here + // for it to survive the form. + schema: JSON.stringify(FLOW_NAME_SCHEMA) }) await callGlobalTool('write_flow', { path: 'f/flows/parent-flow', @@ -4633,6 +5321,419 @@ describe('global AI tools', () => { }) }) + it('test_run_step runs a deployed subflow step past its preprocessor', async () => { + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + path: 'f/flows/deployed-sub', + summary: 'Deployed subflow', + value: { modules: [{ id: 'sub_start', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA + } as any) + await callGlobalTool('write_flow', { + path: 'f/flows/parent-of-deployed', + summary: 'Parent flow', + modules: JSON.stringify([ + { + id: 'call_deployed', + value: { type: 'flow', path: 'f/flows/deployed-sub', input_transforms: {} } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool('test_run_step', { + path: 'f/flows/parent-of-deployed', + stepId: 'call_deployed', + args: { name: 'Ada' } + }) + ) + + expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(JobService.runFlowByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/flows/deployed-sub', + requestBody: { name: 'Ada' }, + skipPreprocessor: true + }) + }) + + // A step is fed by its input transforms, so its arguments are its own and the flow's + // schema describes a different set entirely. Opening the form on the flow's would offer + // fields this job ignores and drop the ones it takes. + it('test_run_step opens the form on the step, not on the flow', async () => { + const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' + await callGlobalTool('write_flow', { + path: 'f/flows/step-form', + summary: 'Step form flow', + // The flow takes `customer`; the step takes `name`. Nothing links the two. + schema: JSON.stringify({ type: 'object', properties: { customer: { type: 'string' } } }), + modules: JSON.stringify([ + { + id: 'format_name', + value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } + } + ]) + }) + + stubInferredProperties({ name: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { + path: 'f/flows/step-form', + stepId: 'format_name', + args: { name: 'Ada', customer: 'acme' } + }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + } + ) + ) + + expect(form.schema.properties).toEqual({ name: { type: 'string' } }) + expect(form.runnableKind).toBe('script') + expect(form.summary).toBe('step "format_name"') + // `customer` is the flow's argument, so the step's form never offered it. + expect(form.args).toEqual({ name: 'Ada' }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { content, language: 'bun', args: { name: 'Grace' } } + }) + }) + + // The step runs the draft script's content, so a form built from the deployed schema + // would offer the arguments of code that is not the code about to run. + it('test_run_step opens a script step on the draft schema, not the deployed one', async () => { + const content = 'export async function main(name: string) {\n\treturn `draft ${name}`\n}' + seedBackendDraft('script', 'f/scripts/drifted', { + path: 'f/scripts/drifted', + summary: 'Drifted', + content, + language: 'bun' + }) + await callGlobalTool('write_flow', { + path: 'f/flows/drifted-step', + summary: 'Drifted step flow', + modules: JSON.stringify([ + { + id: 'call_script', + value: { type: 'script', path: 'f/scripts/drifted', input_transforms: {} } + } + ]) + }) + + // Inferred from the draft's content. Never fetching the deployed script is the point: + // its stored schema describes code this run is not about to execute. + stubInferredProperties({ name: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/drifted-step', stepId: 'call_script', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() + expect(form.schema.properties).toEqual({ name: { type: 'string' } }) + }) + + // No parser emits `password`, so the draft's stored schema is the only thing carrying it. + // Rebuilding the form's fields from the content would offer the secret as a plain text + // box, and the literal typed into it would reach the job's arguments unminted. + it('test_run_step keeps the password marking of a drafted script step', async () => { + seedBackendDraft('script', 'f/scripts/secretful', { + path: 'f/scripts/secretful', + summary: 'Secretful', + content: 'export async function main(token: string) {\n\treturn 1\n}', + language: 'bun', + schema: { + type: 'object', + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + }) + await callGlobalTool('write_flow', { + path: 'f/flows/secretful-step', + summary: 'Secretful step flow', + modules: JSON.stringify([ + { + id: 'call_secretful', + value: { type: 'script', path: 'f/scripts/secretful', input_transforms: {} } + } + ]) + }) + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/secretful-step', stepId: 'call_secretful', args: {} }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + expect(form.schema.properties).toMatchObject({ token: { password: true } }) + }) + + // The entrypoint override is declared by no schema, so it has to be added after the form + // rather than proposed into it — anything that conforms arguments to a schema drops it, + // and the preprocessor then silently runs its `main`. + it('test_run_step keeps the preprocessor entrypoint out of the form and on the job', async () => { + const content = 'export async function preprocessor(event: string) {\n\treturn event\n}' + await callGlobalTool('write_flow', { + path: 'f/flows/preprocessed', + summary: 'Preprocessed flow', + modules: JSON.stringify([{ id: 'start', value: { type: 'identity' } }]), + preprocessor_module: JSON.stringify({ + id: 'preprocessor', + value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } + }) + }) + + vi.mocked(inferArgs).mockClear() + stubInferredProperties({ event: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/preprocessed', stepId: 'preprocessor', args: { event: 'signup' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + // Inferred against the preprocessor entrypoint, not `main`. + expect(vi.mocked(inferArgs).mock.calls[0][3]).toBe('preprocessor') + expect(form.schema.properties).toEqual({ event: { type: 'string' } }) + expect(form.args).toEqual({ event: 'signup' }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + content, + language: 'bun', + args: { _ENTRYPOINT_OVERRIDE: 'preprocessor', event: 'signup' } + } + }) + }) + + // The form IS the consent, so a dismissed one must leave the script unrun. + it('run_script starts no job when the user cancels the form', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/deployed', + summary: 'Deployed', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/deployed', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async () => undefined } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('The user cancelled the run form') + expect(result).toContain('Do not call run_script again') + }) + + // job_args.test.ts owns what each step of the argument pipeline does; this owns that + // run_script still runs them. Delete a call from runThroughForm and every one of those + // unit tests still passes, so one call has to cross all of them here. + it('run_script puts the proposed arguments through the whole pipeline', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/everything', + schema: { + properties: { + count: { type: 'number' }, + ratio: { type: 'number' }, + size: { type: 'number' }, + token: { type: 'string', password: true }, + locked: { type: 'string', default: 'fixed', disabled: true }, + doc: { type: 'string', contentEncoding: 'base64' } + } + } + } as any) + + const bytes = 'QUJD'.repeat(1024) + const statuses: any[] = [] + let shown: Record | undefined + let cleared: string[] | undefined + let reset: string[] | undefined + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { + path: 'f/scripts/everything', + args: { + count: '7', + ratio: 'abc', + size: '$var:u/admin/batch_size', + token: 'hunter2', + locked: 'tampered', + doc: bytes, + force_delete: true + } + }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + requestRunArgs: async (_toolId, form) => { + shown = form.args + cleared = form.clearedKeys + reset = form.resetKeys + // What the user does with the form: attaches the file no model can produce, + // and names a variable for the secret it was not allowed to fill. + return { ...form.args, doc: bytes, token: '$var:u/ada/prod_api_key' } + } + } + ) + ) + + // Coerced, cleared, left alone, reset, dropped and emptied of bytes — every rule reached + // through the tool rather than called directly. The proposed secret is not emptied: it is + // already in the model's own tool call in the same stored record, and PasswordArgInput + // mints whatever the field opens with before the job sees it. + expect(shown).toEqual({ + count: 7, + size: '$var:u/admin/batch_size', + token: 'hunter2', + locked: 'fixed' + }) + expect(cleared).toEqual(['ratio']) + expect(reset).toEqual(['locked']) + + // The bytes belong in the job request and nowhere else: the card is persisted, and a + // file small enough to survive truncation would otherwise reach the model whole. + expect(JobService.runScriptByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/scripts/everything', + requestBody: { + count: 7, + size: '$var:u/admin/batch_size', + locked: 'fixed', + doc: bytes, + token: '$var:u/ada/prod_api_key' + }, + skipPreprocessor: true + }) + expect(result).toContain('does not declare force_delete') + expect(result).toContain('') + // The bytes are the value; the reference is not, and the run page shows it for this + // same job. + expect(result).not.toContain(bytes) + expect(JSON.stringify(statuses)).not.toContain(bytes) + expect(result).toContain('$var:u/ada/prod_api_key') + expect(JSON.stringify(statuses)).toContain('$var:u/ada/prod_api_key') + // Named, or an emptied field reads as the user having deleted the value and the next + // call proposes the same bytes again. + for (const named of ['ratio', 'doc', 'locked']) expect(result).toContain(named) + }) + + // The form is its own confirmation, so it never reaches processToolCall's second gate. + // Plan mode can be switched on while it sits open, and the job must not start. + it('run_script starts no job when plan mode is entered while the form is open', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/noargs', + schema: { properties: {} } + } as any) + + let planning = false + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/noargs', args: {} }, + { + ...toolCallbacks, + isPlanModeActive: () => planning, + requestRunArgs: async (_toolId, form) => { + planning = true + return form.args + } + } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('plan mode is active') + }) + + // A form that reaches the screen mints a proposed secret on mount, which the gate after + // the user answers is too late to unmake — so plan mode arriving during the fetch counts. + it('run_script opens no form when plan mode is entered during the schema fetch', async () => { + let planning = false + vi.mocked(ScriptService.getScriptByPath).mockImplementationOnce(async () => { + planning = true + return { + path: 'f/scripts/pw', + schema: { properties: { token: { type: 'string', password: true } } } + } as any + }) + + let formOpened = false + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/pw', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + isPlanModeActive: () => planning, + requestRunArgs: async (_toolId, form) => { + formOpened = true + return form.args + } + } + ) + + expect(formOpened).toBe(false) + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('plan mode is active') + }) + + it('run_script runs the arguments the user submitted, not the ones proposed', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/greet', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/greet', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async () => ({ name: 'Grace' }) } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/scripts/greet', + requestBody: { name: 'Grace' }, + skipPreprocessor: true + }) + // The model must not assume its proposal is what ran. + expect(result).toContain('Ran with arguments: {"name":"Grace"}') + }) + + // The arguments are already in the call this result answers, and every way the form's + // own differ from the proposed ones has its own clause — so an untouched form has + // nothing to name, and naming it anyway pays for the copy on every later iteration. + it('run_script names the arguments only when the user changed them', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/greet', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/greet', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(result).not.toContain('Ran with arguments') + expect(result).toContain('unedited') + }) + it('test_run_step lists nested step ids when a step is not found', async () => { await callGlobalTool('write_flow', { path: 'f/flows/nested-step-error', @@ -5323,6 +6424,9 @@ describe('session-only preview tools gating', () => { expect(names).not.toContain('list_app_runs') expect(names).not.toContain('search_dom') expect(names).not.toContain('read_dom') + // Not withheld: without it the side panel's only route to a deployed run is the raw + // endpoint, which confirms an opaque request body instead of the arguments. + expect(names).toContain('run_script') // other tools are still present expect(names).toContain('write_script') }) @@ -5335,6 +6439,7 @@ describe('session-only preview tools gating', () => { expect(names).toContain('list_app_runs') expect(names).toContain('search_dom') expect(names).toContain('read_dom') + expect(names).toContain('run_script') // The session set is the full globalTools minus capability-gated tools: // this environment is not Chromium, so take_screenshot is withheld (DOM // capture is only faithful on Blink). search_dom / read_dom are not gated. @@ -5437,13 +6542,16 @@ describe('session-only preview tools gating', () => { }) describe('read_skill', () => { - it('refuses a path the user has not selected, without reading it', async () => { + // Every path is enabled by default now, so the listing is what keeps the tool to + // skills: without it the model could name any resource holding a string `content` + // and have it read back. + it('refuses a path that is not a skill in the workspace, without reading it', async () => { localStorage.clear() userStore.set({ username: 'bob', email: 'bob@windmill.dev', workspace_id: WORKSPACE } as any) const res = await callGlobalTool('read_skill', { path: 'u/someone/private-notes' }) - expect(res).toContain('not one of the skills selected') + expect(res).toContain('not one of the skills available') expect(vi.mocked(ResourceService.getResourceValue)).not.toHaveBeenCalled() }) }) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index e919769e7a..52f4ee39ef 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -20,6 +20,7 @@ import { WebsocketTriggerService } from '$lib/gen' import { createTwoFilesPatch } from 'diff' +import { deepEqual } from 'fast-equals' import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' import { $ScriptLang } from '$lib/gen/schemas.gen' import type { @@ -47,6 +48,16 @@ import { STARTER_RUNNABLE_KEY, type FrameworkKey } from '$lib/components/raw_apps/templates' +import { + coerceArgsToSchema, + dropUndeclaredArgs, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs, + stripFileArgs +} from '$lib/components/job_args' +import { processSecretArgs } from '$lib/components/secretArgUtils' +import { PLAN_MODE_MESSAGES } from '../planModeMessages' import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' import type { RawAppDomQuery } from '$lib/components/raw_apps/rawAppDom' @@ -83,7 +94,7 @@ import { import { searchNpmPackagesTool } from '../script/core' import type { McpServer } from './mcpTools' import { logFeatureUsage } from '$lib/utils/featureUsage' -import { enabledSkillPaths } from '../skills/enabledSkills' +import { isSkillEnabled } from '../skills/enabledSkills' import { listSkillResources, readSkillBody, @@ -114,12 +125,14 @@ import { createToolDef, droppedOptionKeys, createSearchHubScriptsTool, - executeFlowStepTestRun, executeTestRun, findAndReplace, isHubPath, + resolveFlowStepRun, + SPECIAL_MODULE_IDS, type CreatedResourceTriggerKind, type PreviewCardKind, + type RunFormDisplay, type Tool, type ToolCallbacks, type ToolDisplayAction @@ -136,6 +149,7 @@ import type { SessionArtifactsStore } from '../artifacts/artifactsState.svelte' import type { Runnable } from '$lib/components/apps/inputType' import { UserDraft } from '$lib/userDraft.svelte' import { emptySchema } from '$lib/utils' +import type { Schema } from '$lib/common' import { inferArgs } from '$lib/infer' import { resourceRequestSchema, @@ -276,6 +290,8 @@ const ACTIVE_GLOBAL_EDITOR_DRAFTS: readonly { export type GlobalActiveEditorContext = { type: ActiveGlobalEditorType path: string + /** The key the draft is stored under, which `path` leaves behind on a rename. */ + storagePath: string isLiveDraft: true } @@ -889,7 +905,21 @@ const testRunScriptSchema = z.object({ const testRunScriptToolDef = createToolDef( testRunScriptSchema, 'test_run_script', - 'Execute a preview-style test run of a script by path, preferring draft content when it exists.', + 'Execute a preview-style test run of a script by path, preferring draft content when it exists. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.', + { strict: false } +) + +const runScriptSchema = z.object({ + path: z.string().describe('Workspace path of the deployed script to run.'), + args: testRunArgsSchema, + background: backgroundArgSchema, + wait_seconds: waitSecondsArgSchema +}) + +const runScriptToolDef = createToolDef( + runScriptSchema, + 'run_script', + 'Run a DEPLOYED script for real, under the user\'s own permissions. Fill in every argument you can infer: the user gets an argument form prefilled with `args` and decides what runs. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call. A required file is the user\'s to attach, so call this even when you cannot supply one rather than asking in chat. Use only when the user names the deployed version ("the deployed X", "in production", "for real"); otherwise use test_run_script.', { strict: false } ) @@ -903,7 +933,21 @@ const testRunFlowSchema = z.object({ const testRunFlowToolDef = createToolDef( testRunFlowSchema, 'test_run_flow', - 'Execute a preview-style test run of a flow by path, preferring draft content when it exists.', + 'Execute a preview-style test run of a flow by path, preferring draft content when it exists. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.', + { strict: false } +) + +const runFlowSchema = z.object({ + path: z.string().describe('Workspace path of the deployed flow to run.'), + args: testRunArgsSchema, + background: backgroundArgSchema, + wait_seconds: waitSecondsArgSchema +}) + +const runFlowToolDef = createToolDef( + runFlowSchema, + 'run_flow', + 'Run a DEPLOYED flow for real, under the user\'s own permissions. Fill in every argument you can infer: the user gets an argument form prefilled with `args` and decides what runs. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call. A required file is the user\'s to attach, so call this even when you cannot supply one rather than asking in chat. Use only when the user names the deployed version ("the deployed X", "in production", "for real"); otherwise use test_run_flow.', { strict: false } ) @@ -918,7 +962,7 @@ const testRunStepSchema = z.object({ const testRunStepToolDef = createToolDef( testRunStepSchema, 'test_run_step', - 'Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists.', + "Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists. `args` are the step's OWN inputs, not the flow's: a step is normally fed by its input transforms, so send what that step's code takes, not what the flow takes. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.", { strict: false } ) @@ -1322,7 +1366,7 @@ ${pipelineBullet} : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. - For a Windmill operation no other tool covers (workers, queue state, a run's args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. -- runScriptByPath / runFlowByPath from the API catalog run the DEPLOYED version of an item. Use them only when the user explicitly asks to run the deployed version, and read the item with read_workspace_item version: "deployed" first so the arguments match the deployed input schema (a draft may have different inputs). To test something you are editing or just wrote, always use test_run_script, test_run_flow, or test_run_step — they run the draft. +- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script or run_flow only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For those two, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema. test_run_script, test_run_flow, test_run_step, run_script and run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. test_run_step's form is the step's own inputs, not the flow's. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ @@ -1373,7 +1417,7 @@ Data Tables: ? ` Skills: -- Skills are reusable instruction sets the user selected for this chat, each covering a specific kind of task. The available skills are listed below by resource path and description. +- Skills are reusable instruction sets available in this workspace, each covering a specific kind of task. The available skills are listed below by resource path and description. - When a user's request matches a skill's description, call read_skill with its exact path to load the full instructions BEFORE acting, then follow them. ${skills.map((s) => `- ${s.path}: ${s.description}`).join('\n')}` : '' @@ -2340,23 +2384,17 @@ export type ChatCommandItem = { } /** - * The skills this user turned on in this workspace, for the global system prompt. - * A readable `ai_skill` resource is only a candidate — enabling one is a personal - * choice, since each enabled skill spends context on every turn. + * The skills in play in this workspace, for the global system prompt: every + * readable `ai_skill` resource except the ones this user turned off. */ export async function loadWorkspaceSkills(workspace: string): Promise { if (!workspace) return [] try { - const enabled = new Set(enabledSkillPaths(workspace)) - if (enabled.size === 0) return [] - // Filtered against what is actually readable now, so a skill that was - // deleted or whose folder access was revoked drops out instead of being - // advertised to the model as something read_skill can load. // A truncated listing still carries most of the workspace, and the drawer is // where that is surfaced; dropping everything here would silently empty the // Skills section instead. return (await listSkillResources(workspace)).skills - .filter((s) => enabled.has(s.path)) + .filter((s) => isSkillEnabled(workspace, s.path)) .map(({ path, name, description }) => ({ path, name, @@ -2382,24 +2420,28 @@ export const readSkillTool: Tool<{}> = { def: createToolDef( readSkillSchema, 'read_skill', - 'Load the full instructions for a selected AI skill by resource path. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' + 'Load the full instructions for an AI skill by resource path. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' ), planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = readSkillSchema.parse(args) const name = skillNameFromPath(parsed.path) - // The prompt lists only selected skills, but the tool takes a path the model - // composed, so the selection is enforced here too rather than assumed. Without - // it the tool reads any resource holding a string `content` — the user's own - // access, but not what "load a selected skill" says it does. - if (!enabledSkillPaths(workspace).includes(parsed.path)) { - toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not selected` }) - return `"${parsed.path}" is not one of the skills selected for this chat. Only the paths listed under "Skills" in the system prompt can be read.` - } toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${name}"...` }) try { - // Bounded here rather than in the reader: any `ai_skill` resource can be - // selected, including ones written through git sync or the resource editor + // The prompt lists the skills in play, but the tool takes a path the model + // composed, so what may be read is checked here rather than assumed. Against + // the listing, not just the off-switch: any other path is now enabled too, + // and without this the tool reads any resource holding a string `content` — + // the user's own access, but not what "load a skill" says it does. + const isSkill = (await listSkillResources(workspace)).skills.some( + (s) => s.path === parsed.path + ) + if (!isSkill || !isSkillEnabled(workspace, parsed.path)) { + toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not available` }) + return `"${parsed.path}" is not one of the skills available to this chat. Only the paths listed under "Skills" in the system prompt can be read.` + } + // Bounded here rather than in the reader: any `ai_skill` resource is in play, + // including ones written through git sync or the resource editor // that never passed the authoring form's limits, and an unbounded body // would exhaust the context on one tool call. The editor reads the same // resource untruncated, so opening a long skill cannot rewrite it short. @@ -2408,7 +2450,7 @@ export const readSkillTool: Tool<{}> = { MAX_SKILL_INSTRUCTIONS_LENGTH ) toolCallbacks.setToolStatus(toolId, { content: `Read skill "${name}"` }) - // Whether a selected skill is actually reached for. No key: the path is + // Whether a skill is actually reached for. No key: the path is // workspace-authored text. logFeatureUsage('ai_session', 'skill_read', { workspace }) return `Skill: ${parsed.path}\n\nInstructions:\n${instructions}` @@ -3384,7 +3426,8 @@ export const globalTools: Tool<{}>[] = [ draftCountByType.set(draft.type, count + 1) byKey.set(getWorkspaceItemKey(draft.type, draft.path, draft.triggerKind), { ...draft, - value: undefined + value: undefined, + schema: undefined }) } } @@ -3639,33 +3682,68 @@ export const globalTools: Tool<{}>[] = [ const parsed = testRunScriptSchema.parse(ctx.args) return testRunScriptByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => `Run a test of ${pathLeaf(args?.path, 'the script')}`, + // No requiresConfirmation: the argument form is the confirmation, and the bypass posture + // answers it with what the form opened with — a decision made for the user, so the + // posture's own list has to name it. One thing does run before Run: see the note on + // the form's SchemaForm. + bypassedByAutoAccept: true, + confirmationMessage: 'Run a test of a script', + streamingLabel: 'Preparing the test form...', queuedLabel: (args) => `Test ${args?.path ?? 'the script'}`, showDetails: true, autoCollapseDetails: false }, + { + def: runScriptToolDef, + fn: async (ctx) => { + const parsed = runScriptSchema.parse(ctx.args) + return runDeployedScript(parsed, ctx) + }, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + confirmationMessage: 'Run a deployed script', + streamingLabel: 'Preparing the run form...', + queuedLabel: (args) => `Run ${args?.path ?? 'a script'}`, + showDetails: true, + autoCollapseDetails: false + }, { def: testRunFlowToolDef, fn: async (ctx) => { const parsed = testRunFlowSchema.parse(ctx.args) return testRunFlowByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => `Run a test of ${pathLeaf(args?.path, 'the flow')}`, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the test form...', + confirmationMessage: 'Run a test of a flow', queuedLabel: (args) => `Test ${args?.path ?? 'the flow'}`, showDetails: true, autoCollapseDetails: false }, + { + def: runFlowToolDef, + fn: async (ctx) => { + const parsed = runFlowSchema.parse(ctx.args) + return runDeployedFlow(parsed, ctx) + }, + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the run form...', + confirmationMessage: 'Run a deployed flow', + queuedLabel: (args) => `Run ${args?.path ?? 'a flow'}`, + showDetails: true, + autoCollapseDetails: false + }, { def: testRunStepToolDef, fn: async (ctx) => { const parsed = testRunStepSchema.parse(ctx.args) return testRunFlowStepByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => - `Run a test of step "${args?.stepId ?? ''}" in ${pathLeaf(args?.path, 'the flow')}`, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the test form...', + confirmationMessage: 'Run a test of a flow step', queuedLabel: (args) => `Test step "${args?.stepId ?? ''}" of ${args?.path ?? 'the flow'}`, showDetails: true, autoCollapseDetails: false @@ -4282,8 +4360,8 @@ export const SESSION_PREVIEW_TOOL_NAMES = new Set([ /** * The global tool set for a given chat: the full `globalTools` for a session - * chat, or `globalTools` minus the session-only preview tools for the regular - * global side-panel chat. + * chat, or `globalTools` minus the preview tools for the regular global + * side-panel chat. */ export function globalToolsFor({ sessionPreview }: { sessionPreview: boolean }): Tool<{}>[] { const tools = sessionPreview @@ -4324,7 +4402,8 @@ type WriteDraftCtx = { export type SessionToolHelpers = { sessionId?: string } export type GlobalToolHelpers = SessionToolHelpers & { - testActiveFlow?: (args?: Record) => Promise + /** Runs the flow editor mounted on `storagePath`, if one is. */ + testActiveFlow?: (storagePath: string, args?: Record) => Promise attachedFiles?: AttachedFilesStore // Read/write the user-level Global instructions. `setUserInstructions` persists the // value and rebuilds the system message so the change applies on the next chat-loop @@ -4353,15 +4432,21 @@ function operatingWorkspaceFromHelpers(helpers: unknown): string | undefined { return (helpers as GlobalToolHelpers | undefined)?.operatingWorkspace } -function activeFlowTestFromCtx( +// Drive a live editor only for the flow on screen: several can be open at once (session tabs), +// and a run painted into a background tab is a side effect the user never sees. Undefined +// sends the caller to a preview run, which reports into the chat alone. The hook is bound to +// that editor's storage path — the same key reads and edits of `path` resolve through, so a +// staged rename cannot send the run to a different editor than the one being edited. +function liveFlowTestHookFromCtx( ctx: { workspace: string; helpers?: unknown }, path: string -): GlobalToolHelpers['testActiveFlow'] | undefined { +): ((args?: Record) => Promise) | undefined { const activeEditor = getActiveGlobalEditorContext(ctx.workspace) if (activeEditor?.type !== 'flow' || activeEditor.path !== path) { return undefined } - return (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + const testActiveFlow = (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + return testActiveFlow && ((args) => testActiveFlow(activeEditor.storagePath, args)) } export type OpenPreviewHandler = (req: { @@ -4952,11 +5037,14 @@ const SCRIPT_SPEC: WriteSpec = { language: args.language, kind: 'script' } - // Infer the arg schema from the content at save time, like the editor does, - // so the persisted draft is the single source of truth at deploy. Keep the - // previous schema (or empty) on failure rather than blanking it. + // Into the schema the base carries, as the editor does at save: `inferArgs` re-seeds + // each arg from the properties it is handed, and those are the only copy of + // `password`, enums, formats and titles — no parser emits them. A clone, so a parse + // failure leaves the previous schema rather than half of one. try { - const schema = emptySchema() + const schema = structuredClone( + draft.schema?.properties ? draft.schema : emptySchema() + ) as Schema await inferArgs(draft.language, draft.content, schema) draft.schema = schema } catch (e) { @@ -5116,16 +5204,56 @@ function writeVariableDraft(args: WriteVariableArgs, ctx: WriteDraftCtx): Promis async function loadScriptForEdit( path: string, workspace: string -): Promise<{ content: string; language: ScriptLang; summary?: string }> { +): Promise<{ + content: string + language: ScriptLang + summary?: string + schema?: Record +}> { const draft = await getGlobalDraft(workspace, 'script', path) if (draft) { if (typeof draft.value !== 'string' || !draft.language) { throw new Error(`Draft script "${path}" is missing content or language.`) } - return { content: draft.value, language: draft.language, summary: draft.summary } + return { + content: draft.value, + language: draft.language, + summary: draft.summary, + schema: draft.schema as Record | undefined + } } const script = await ScriptService.getScriptByPath({ workspace, path }) - return { content: script.content, language: script.language, summary: script.summary } + return { + content: script.content, + language: script.language, + summary: script.summary, + schema: script.schema as Record | undefined + } +} + +/** The fields a test form offers, for code that may never have been deployed. The stored + * schema wins wherever it declares fields — a draft's or the deployed script's; one that + * declares nothing, or that speaks for an entrypoint other than the one about to run, is + * inferred here from the content instead. */ +async function schemaForTestRun(script: { + content: string + language: ScriptLang + schema?: Record + /** A preprocessor takes the arguments of its own entrypoint, not of `main`. */ + entrypoint?: 'preprocessor' +}): Promise> { + // Emptily declared is not declared: a stored `properties: {}` means the schema predates + // the arguments the code now takes, so infer rather than offer a form with no fields. + // A stored schema speaks for one entrypoint, so it can never answer for an override. + if (!script.entrypoint && Object.keys(script.schema?.properties ?? {}).length > 0) + return script.schema! + const schema = emptySchema() + try { + await inferArgs(script.language, script.content, schema, script.entrypoint) + } catch (e) { + console.error('Failed to infer script schema for the test run form', e) + } + return schema as unknown as Record } async function editScript( @@ -5151,18 +5279,21 @@ async function editScript( async function loadFlowDraftValue( path: string, workspace: string -): Promise<{ flow: FlowDraftValue; summary?: string }> { + // `isDraft` says which of the two this came from. Reported here because the draft lookup + // is a request of its own: a caller that needs to know would otherwise repeat it. +): Promise<{ flow: FlowDraftValue; summary?: string; isDraft: boolean }> { const draft = await getGlobalDraft(workspace, 'flow', path) if (draft) { if (draft.value === undefined || typeof draft.value === 'string') { throw new Error(`Draft flow "${path}" has no value.`) } - return { flow: draft.value as FlowDraftValue, summary: draft.summary } + return { flow: draft.value as FlowDraftValue, summary: draft.summary, isDraft: true } } const flow = await FlowService.getFlowByPath({ workspace, path }) return { flow: { value: flow.value, schema: flow.schema, groups: flow.value.groups ?? null }, - summary: flow.summary + summary: flow.summary, + isDraft: false } } @@ -5333,30 +5464,42 @@ function flowDraftValueForPreview(flowDraft: FlowDraftValue): FlowValue { async function loadScriptForFlowStep( moduleValue: { path: string; hash?: string }, workspace: string -): Promise<{ content: string; language: ScriptLang }> { +): Promise<{ content: string; language: ScriptLang; schema?: Record }> { const draft = await getGlobalDraft(workspace, 'script', moduleValue.path) if (draft) { if (typeof draft.value !== 'string' || !draft.language) { throw new Error(`Draft script "${moduleValue.path}" is missing content or language.`) } - return { content: draft.value, language: draft.language } + return { + content: draft.value, + language: draft.language, + // The draft's own: no parser emits `password`, so a schema rebuilt from the content + // would offer a secret argument as a plain field and take the literal into the job. + schema: draft.schema as Record | undefined + } } const script = moduleValue.hash ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) - return { content: script.content, language: script.language } + return { + content: script.content, + language: script.language, + schema: script.schema as Record | undefined + } } -async function loadDraftFlowPreviewValue( +async function loadSubflowForFlowStep( path: string, workspace: string -): Promise { - if (!(await getGlobalDraft(workspace, 'flow', path))) { - return undefined - } +): Promise<{ previewValue?: FlowValue; schema?: Record }> { const nestedFlow = await loadFlowDraftValue(path, workspace) - return flowDraftValueForPreview(nestedFlow.flow) + return { + // Only a draft is previewed; a deployed subflow is run by path, as its parent flow + // would run it. The schema describes whichever of the two that leaves. + previewValue: nestedFlow.isDraft ? flowDraftValueForPreview(nestedFlow.flow) : undefined, + schema: nestedFlow.flow.schema ?? undefined + } } // Leaf of a workspace path (last segment), for human-readable confirmation @@ -5370,90 +5513,435 @@ async function testRunScriptByPath( args: z.infer, ctx: WriteDraftCtx ): Promise { - const { workspace, toolId, toolCallbacks } = ctx + const { workspace } = ctx const script = await loadScriptForEdit(args.path, workspace) - const testArgs = normalizeTestRunArgs(args.args) + const schema = await schemaForTestRun(script) - return executeTestRun({ - jobStarter: () => - JobService.runScriptPreview({ - workspace, - requestBody: { - path: args.path, - content: script.content, - args: testArgs, - language: script.language - } - }), + return runThroughForm( + { + path: args.path, + schema, + summary: script.summary, + kind: 'test', + code: script.content, + lang: script.language, + // Never "deployed" here: the code about to run is the draft the model is still + // writing, and a line telling it to re-read the deployed schema would send it + // to the wrong version. + schemaNoun: 'script', + toolName: 'test_run_script', + proposed: args.args, + startMessage: `Running test for script "${args.path}"...`, + contextName: 'script', + // Its own loop: the model is told to test and iterate, so the posture answers the + // form with what it opened with rather than parking the loop on a card. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runScriptPreview({ + workspace, + requestBody: { + path: args.path, + content: script.content, + args: submitted, + language: script.language + } + }) + }, + ctx + ) +} + +/** The "do not call again" half is load-bearing: without it the model re-proposes the + * call, which re-opens the form the user just dismissed, and Stop becomes their only + * way out. */ +const runFormCancelled = (toolName: string, noun: string) => + `The user cancelled the run form. The ${noun} did NOT run. Do not call ${toolName} again unless the user asks for it.` + +/** The model only needs to see what the user changed, and nothing bounds an object or + * array argument the form let them paste into. */ +const MAX_SUBMITTED_ARGS_LENGTH = 4000 + +/** The card's own copy is bounded separately, and far higher: it is what the details pane + * renders, and JobArgs stops rendering the JSON in full at this size regardless. */ +const MAX_PERSISTED_ARGS_LENGTH = 100_000 + +/** One run through an argument form: conform what the model proposed to the schema of the + * version about to run, open the form on it, then run whatever came back. Both tools that + * run a script are this, differing only in where the schema comes from and how the job + * starts — so the user meets one card whichever they asked for. */ +type FormRunSpec = { + path: string + schema: Record + summary?: string + kind: 'run' | 'test' + /** The code a test run is about to preview, so its form can offer the same dynamic-option + * pickers the script editor's test panel does. Omitted for a deployed run, which names a + * path instead. */ + code?: string + lang?: ScriptLang + /** How the lines the model reads back name the version this ran: telling it to re-read + * the "deployed schema" of a draft would send it to the wrong code. */ + schemaNoun: string + toolName: string + proposed: Record | null | undefined + startMessage: string + /** What runs: the jobs-tray kind, and the noun the card's own prose reads. */ + contextName: 'script' | 'flow' + /** What the lines the model reads back call the thing that ran. Defaults to + * `contextName`, which a flow step is not: it runs a script or a subflow but is neither. */ + noun?: string + /** What names the run where its path would not: the jobs-tray row, and the two + * background-job sentences the model reads. Those quote it, so it carries none. */ + label?: string + /** Whether the bypass posture may answer this form with what it opened with. */ + autoAcceptable?: boolean + background?: boolean + detachAfterMs?: number + startJob: (submitted: Record) => Promise +} + +async function runThroughForm(spec: FormRunSpec, ctx: WriteDraftCtx): Promise { + const { workspace, toolId, toolCallbacks } = ctx + // Asked of the posture, not of the tool: every run tool is auto-acceptable, so a + // host with no form would otherwise run one on the model's arguments alone, in any + // posture. What a bypass answers is a decision the user already made; without it there + // is no consent to be had here and nothing to fall back on. + const postureAnswers = Boolean( + spec.autoAcceptable && toolCallbacks.shouldAutoAcceptToolConfirmations?.(spec.toolName) + ) + const noun = spec.noun ?? spec.contextName + if (!toolCallbacks.requestRunArgs && !postureAnswers) { + return `This chat cannot show a run form, so a ${noun} cannot be run from here.` + } + + // processToolCall gates plan mode once, before the schema fetch, and this form is its own + // confirmation so it never reaches that gate again. Repeated wherever a write follows: a + // mounted field mints on its own, which no later gate can unmake. + const blockedByPlanMode = (): string | undefined => { + if (!toolCallbacks.isPlanModeActive?.()) return undefined + toolCallbacks.onToolBlockedByPlanMode?.() + toolCallbacks.setToolStatus(toolId, { + content: PLAN_MODE_MESSAGES.blockedLabel, + isLoading: false, + isStreamingArguments: false, + error: PLAN_MODE_MESSAGES.blockedResult, + blockedByPlanMode: true + }) + return PLAN_MODE_MESSAGES.blockedResult + } + const blockedBeforeForm = blockedByPlanMode() + if (blockedBeforeForm) return blockedBeforeForm + + const schema = spec.schema + // Whether to ask is the only question decided here. What a mounted field would hold — a + // default, a synthesised empty, whether Run lights up — is the form's own business: any + // second derivation of it here can start a run the form itself would refuse. + const autoAccepted = postureAnswers + const strippedKeys: string[] = [] + const coerced = autoAccepted + ? undefined + : coerceArgsToSchema(normalizeTestRunArgs(spec.proposed), schema) + let proposed: Record + let resetKeys: string[] + let undeclaredKeys: string[] + if (coerced) { + resetKeys = coerced.resetKeys + undeclaredKeys = coerced.undeclaredKeys + // Left as the model proposed it, minted by the widget the field mounts: a reference put + // here instead would be normalised away by the nested form an object secret renders as. + proposed = stripFileArgs(coerced.args, schema as any, strippedKeys) + } else { + // Both rules hold against every caller, not only the ones a form stands in front of: + // an undeclared argument has no field anywhere, and a disabled one is nobody's to set. + // Without the rest of the coercion, which answers what a mounted widget would show. + const declared = dropUndeclaredArgs(normalizeTestRunArgs(spec.proposed), schema) + undeclaredKeys = declared.undeclaredKeys + const enforced = enforceDisabledDefaults(declared.args, schema) + resetKeys = enforced.resetKeys + // In the widget's stead: with no form there is no PasswordArgInput to turn a proposed + // secret into a reference, and the job's arguments outlive the run. + try { + proposed = await processSecretArgs(enforced.args, schema as any, workspace) + } catch (e) { + const message = `Failed to store the sensitive arguments of "${spec.path}": ${e}` + toolCallbacks.setToolStatus(toolId, { + content: message, + isLoading: false, + isStreamingArguments: false, + error: message + }) + return message + } + } + const form: RunFormDisplay = { + path: spec.path, + summary: spec.summary || undefined, + kind: spec.kind, + runnableKind: spec.contextName, + schema: autoAccepted ? undefined : schema, + code: autoAccepted ? undefined : spec.code, + lang: autoAccepted ? undefined : spec.lang, + submitted: autoAccepted || undefined, + args: proposed, + clearedKeys: coerced?.clearedKeys.length ? coerced.clearedKeys : undefined, + resetKeys: resetKeys.length ? resetKeys : undefined, + strippedKeys: strippedKeys.length ? strippedKeys : undefined + } + + // Files only: nothing rewrites `runForm.args` after this, so bytes left in it outlive the + // size guard that covers `parameters`. + const persisted = { ...form, args: redactFileArgs(proposed, schema as any) } + + toolCallbacks.setToolStatus(toolId, { + content: autoAccepted + ? spec.startMessage + : `Waiting for you to confirm the arguments of "${spec.path}"`, + runForm: persisted, + // Not the raw tool-call arguments: the card settles on what the form opened with. + // Only settles it — the raw proposal still renders while the call streams in. + parameters: persisted.args, + isLoading: true + }) + + // `form`, not `persisted`: a password field mints from what it opens with. + const submitted = toolCallbacks.requestRunArgs + ? await toolCallbacks.requestRunArgs(toolId, form, { autoAccepted }) + : proposed + if (!submitted) { + toolCallbacks.setToolStatus(toolId, { + content: `Run of "${spec.path}" cancelled by user`, + isLoading: false, + isStreamingArguments: false, + error: 'Cancelled by user', + declinedByUser: true + }) + return runFormCancelled(spec.toolName, noun) + } + + const blockedBeforeRun = blockedByPlanMode() + if (blockedBeforeRun) return blockedBeforeRun + + // Every job leaves through here, so this is where a sensitive argument becomes a reference: + // the form mints as the user types and the bypass mints in its stead, but a host answering + // the form its own way — the eval harness does — would hand over a literal. Idempotent, so + // the two that already minted pay a walk and no round trip. + let toRun: Record + try { + toRun = await processSecretArgs(submitted, schema as any, workspace) + } catch (e) { + const message = `Failed to store the sensitive arguments of "${spec.path}": ${e}` + toolCallbacks.setToolStatus(toolId, { + content: message, + isLoading: false, + isStreamingArguments: false, + error: message + }) + return message + } + + // The card's details pane must show what ran, not what was proposed. Bytes are marked by + // size because the card is persisted; everything else stands as the run page shows it for + // the same job. + const forCard = redactFileArgs(toRun, schema as any) + // The transcript is re-cloned into IndexedDB on every save and a form carries whatever was + // pasted into it, so past what the pane would render the card reads the arguments off the + // job instead. Only once there is a job to read them from: substituting the marker any + // earlier would leave a run that never started showing nothing but the marker. + const oversized = JSON.stringify(forCard).length > MAX_PERSISTED_ARGS_LENGTH + if (!oversized) { + toolCallbacks.setToolStatus(toolId, { parameters: forCard }) + } + + const outcome = await executeTestRun({ + jobStarter: async () => { + const jobId = await spec.startJob(toRun) + // The form's own submitted flag flips a round trip earlier, when the user presses + // Run; only from here is there a job for a stopped turn to say it left running. + toolCallbacks.markRunFormStarted?.(toolId) + if (oversized) { + toolCallbacks.setToolStatus(toolId, { parameters: { reason: 'WINDMILL_TOO_BIG' } }) + } + return jobId + }, workspace, toolCallbacks, toolId, - startMessage: `Running test for script "${args.path}"...`, - contextName: 'script', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path + startMessage: spec.startMessage, + contextName: spec.contextName, + actionNoun: spec.kind === 'test' ? 'test' : 'run', + background: spec.background, + detachAfterMs: spec.detachAfterMs, + label: spec.label ?? spec.path }) + + const schemaNoun = `${spec.schemaNoun} schema` + // Only what the form could make no reading of: a wrong-typed value it can read is + // converted silently, since the field then shows what the run carries and there is + // nothing to report. + const clearedKeys = coerced?.clearedKeys ?? [] + const cleared = clearedKeys.length + ? `\nThe ${schemaNoun} declares ${clearedKeys.join(', ')}, but you sent ${clearedKeys.length > 1 ? 'them in shapes' : 'it in a shape'} with no reading in the declared ${clearedKeys.length > 1 ? 'types' : 'type'}, so the ${clearedKeys.length > 1 ? 'fields opened' : 'field opened'} empty and the run did not carry ${clearedKeys.length > 1 ? 'them' : 'it'}. Re-read the input schema and match ${clearedKeys.length > 1 ? 'their declared types' : 'its declared type'}.` + : '' + const reset = resetKeys.length + ? `\nThe ${schemaNoun} disables ${resetKeys.join(', ')}, so the run used ${resetKeys.length > 1 ? 'their defaults' : 'its default'} rather than the proposed ${resetKeys.length > 1 ? 'values' : 'value'}. Do not propose ${resetKeys.length > 1 ? 'them' : 'it'} again.` + : '' + // Nothing renders these, so the model is the only one who can be told they went nowhere. + const undeclared = undeclaredKeys.length + ? `\nThe ${schemaNoun} does not declare ${undeclaredKeys.join(', ')}, so ${undeclaredKeys.length > 1 ? 'they were' : 'it was'} not sent — no run form in Windmill offers a field the schema does not name. Re-read the input schema and use the arguments it declares.` + : '' + // Otherwise an emptied field reads as the user having deleted it, and the next call + // proposes the same bytes again. + const stripped = strippedKeys.length + ? `\n${strippedKeys.join(', ')} ${strippedKeys.length > 1 ? 'are file arguments' : 'is a file argument'}, so the form opened ${strippedKeys.length > 1 ? 'them' : 'it'} empty for the user to attach. ${strippedKeys.length > 1 ? 'They are' : 'It is'} theirs to provide, not yours: do not propose ${strippedKeys.length > 1 ? 'them' : 'it'} again.` + : '' + // Redacted for the model alone: what it proposed is already in its own tool call, but a + // secret the user typed into the form would be entering its context here. + const redacted = redactFileArgs(redactSecretArgs(toRun, schema as any), schema as any) + const submittedJson = JSON.stringify(redacted) + const shown = + submittedJson.length > MAX_SUBMITTED_ARGS_LENGTH + ? submittedJson.slice(0, MAX_SUBMITTED_ARGS_LENGTH) + '... (truncated)' + : submittedJson + // Naming them costs a copy of arguments already in the call above, and cleared/reset/ + // stripped name every way the form's own differ from the proposed ones — so only what + // the user changed is news. + const ran = deepEqual(redacted, proposed) + ? 'Ran with the arguments the form opened with, unedited.' + : `Ran with arguments: ${shown}` + return `${ran}${cleared}${reset}${stripped}${undeclared}\n${outcome}` +} + +async function runDeployedScript( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace } = ctx + // No getDraft: this runs the script as it is live, so the form has to offer the + // inputs the live version accepts and not a draft's. + const script = await ScriptService.getScriptByPath({ workspace, path: args.path }) + return runThroughForm( + { + path: args.path, + schema: (script.schema as Record) ?? {}, + summary: script.summary, + kind: 'run', + schemaNoun: 'deployed', + toolName: 'run_script', + proposed: args.args, + startMessage: `Running "${args.path}"...`, + contextName: 'script', + // Bypassable like a test run: the posture is the user's standing answer, and a form + // it parks on is a card nobody is watching. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runScriptByPath({ + workspace, + path: args.path, + requestBody: submitted, + // As the script's own run page does: the form fills the main input schema, and a + // preprocessor would take these arguments for a webhook body and hand the script + // its own output instead. + skipPreprocessor: true + }) + }, + ctx + ) +} + +async function runDeployedFlow( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace } = ctx + // No live editor is driven here as a test run drives one: that editor holds the draft, and a + // deployed run painted into its graph would show steps that are not the ones running. + const flow = await FlowService.getFlowByPath({ workspace, path: args.path }) + return runThroughForm( + { + path: args.path, + schema: (flow.schema as Record) ?? {}, + summary: flow.summary, + kind: 'run', + // No code/lang: the dynamic-option pickers come from the deployed flow, not an inline copy. + schemaNoun: 'deployed', + toolName: 'run_flow', + proposed: args.args, + startMessage: `Running "${args.path}"...`, + contextName: 'flow', + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runFlowByPath({ + workspace, + path: args.path, + requestBody: submitted, + // As the flow's own run page does: the form fills the main input schema, and a + // preprocessor would take these arguments for a webhook body and hand the flow + // its own output instead. + skipPreprocessor: true + }) + }, + ctx + ) } async function testRunFlowByPath( args: z.infer, ctx: WriteDraftCtx ): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const testArgs = normalizeTestRunArgs(args.args) - const testActiveFlow = activeFlowTestFromCtx(ctx, args.path) + const { workspace } = ctx + // The schema must be in hand before the form is built, and the value rides along from the + // same read so the fields and the previewed flow are one version. With an editor open on + // this path this reads its in-memory cell rather than the network. + const flow = await loadFlowDraftValue(args.path, workspace) + const schema = (flow.flow.schema as Record | null | undefined) ?? {} - if (testActiveFlow) { - return executeTestRun({ - jobStarter: async () => { - const jobId = await testActiveFlow(testArgs) + return runThroughForm( + { + path: args.path, + schema, + summary: flow.summary, + kind: 'test', + // A flow's dynamic-option pickers are one script stored on the schema itself, which is + // where the editor's own test form reads them from (FlowPreviewContent). + code: schema['x-windmill-dyn-select-code'], + lang: schema['x-windmill-dyn-select-lang'], + // Never "deployed": a test run previews the draft, so a line sending the model to the + // deployed schema would name the wrong version. + schemaNoun: 'flow', + toolName: 'test_run_flow', + proposed: args.args, + startMessage: `Starting flow test run for "${args.path}"...`, + contextName: 'flow', + // The model is told to test and iterate, so the bypass posture answers the form. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: async (submitted) => { + // An open editor runs its own in-memory flow and paints the run in its graph. + // Resolved here rather than before the form: the form waits as long as the user + // does, and the editor on screen when they press Run is the one it belongs in. + const jobId = await liveFlowTestHookFromCtx(ctx, args.path)?.(submitted) if (jobId) { return jobId } - - const flow = await loadFlowDraftValue(args.path, workspace) return JobService.runFlowPreview({ workspace, requestBody: { path: args.path, value: flowDraftValueForPreview(flow.flow), - args: testArgs + args: submitted } }) - }, - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) - } - - const flow = await loadFlowDraftValue(args.path, workspace) - - return executeTestRun({ - jobStarter: () => - JobService.runFlowPreview({ - workspace, - requestBody: { - path: args.path, - value: flowDraftValueForPreview(flow.flow), - args: testArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) + } + }, + ctx + ) } async function testRunFlowStepByPath( @@ -5462,21 +5950,61 @@ async function testRunFlowStepByPath( ): Promise { const { workspace, toolId, toolCallbacks } = ctx const flow = await loadFlowDraftValue(args.path, workspace) - const flowValue = flowDraftValueForPreview(flow.flow) - const testArgs = normalizeTestRunArgs(args.args) - - return executeFlowStepTestRun({ - flowValue, + const resolved = await resolveFlowStepRun({ + flowValue: flowDraftValueForPreview(flow.flow), stepId: args.stepId, - args: testArgs, workspace, toolCallbacks, toolId, - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), loadScript: loadScriptForFlowStep, - loadFlowPreviewValue: loadDraftFlowPreviewValue + loadSubflow: loadSubflowForFlowStep }) + + // The module resolution landed on, not the id that was asked for: the job's entrypoint + // override reads the same value, and a form built for the other entrypoint offers fields + // the run will not take. + const isPreprocessor = resolved.module.id === SPECIAL_MODULE_IDS.PREPROCESSOR + // The step's own inputs, never the flow's: a step is fed by its input transforms, so the + // flow's schema names arguments this job would ignore and omits the ones it takes. + const schema = + resolved.code != undefined && resolved.lang + ? await schemaForTestRun({ + content: resolved.code, + language: resolved.lang, + schema: resolved.schema, + entrypoint: isPreprocessor ? 'preprocessor' : undefined + }) + : (resolved.schema ?? {}) + + const stepSummary = resolved.module.summary + return runThroughForm( + { + // The flow: a step has no path of its own, and this is what the status and cancel + // lines quote back, so it has to name something the reader can go and open. The + // step itself is named by the summary below. + path: args.path, + schema, + summary: stepSummary ? `step "${args.stepId}": ${stepSummary}` : `step "${args.stepId}"`, + kind: 'test', + code: resolved.code ?? schema['x-windmill-dyn-select-code'], + lang: resolved.lang ?? schema['x-windmill-dyn-select-lang'], + // Never "deployed": a step test previews the draft flow, and the step's target may + // itself be a draft. + schemaNoun: 'step', + toolName: 'test_run_step', + proposed: args.args, + startMessage: resolved.startMessage, + contextName: resolved.runnableKind, + noun: 'step', + label: `step ${args.stepId}`, + // The model is told to test and iterate, so the bypass posture answers the form. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: resolved.startJob + }, + ctx + ) } async function initApp( @@ -7798,9 +8326,10 @@ export function getActiveGlobalEditorContext( ): GlobalActiveEditorContext | undefined { for (const { itemKind, type } of ACTIVE_GLOBAL_EDITOR_DRAFTS) { const liveDraft = UserDraft.getLiveEditorDraft(itemKind, { workspace }) - const path = liveDraft?.effectivePath || liveDraft?.storagePath + if (!liveDraft) continue + const path = liveDraft.effectivePath || liveDraft.storagePath if (!path) continue - return { type, path, isLiveDraft: true } + return { type, path, storagePath: liveDraft.storagePath, isLiveDraft: true } } } diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts index a155115d28..9e1b46589c 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts @@ -4,7 +4,7 @@ const { getMcpToolsMock, callMcpToolMock, listResourceMock, session } = vi.hoist getMcpToolsMock: vi.fn(), callMcpToolMock: vi.fn(), listResourceMock: vi.fn(), - session: { email: 'first@windmill.dev' } + session: { email: 'first@windmill.dev', workspace_id: 'test-ws', username: 'hugo', pgroups: [] } })) vi.mock('../shared', () => ({ @@ -228,7 +228,7 @@ describe('call results', () => { }) describe('search_mcp_tools', () => { - it('returns compact summaries without the full input schemas', async () => { + it('returns each match with its full input schema', async () => { const result = await run('search_mcp_tools', { query: 'issue' }) expect(result.matches).toEqual([ { @@ -236,7 +236,7 @@ describe('search_mcp_tools', () => { tool: 'get_issue', description: 'Get details of a GitHub issue', mode: 'read', - params: ['owner', 'repo'] + inputSchema: TOOLS[0].inputSchema } ]) }) @@ -330,6 +330,64 @@ describe('result size cap', () => { expect(callMcpToolMock).not.toHaveBeenCalled() }) + it('drops the lowest-ranked schemas before dropping the matches themselves', async () => { + getMcpToolsMock.mockResolvedValue( + ['issue_a', 'issue_b', 'issue_c'].map((name) => ({ + name, + description: 'An issue tool', + inputSchema: { + type: 'object', + properties: { body: { type: 'string', description: 'x'.repeat(9_000) } } + }, + annotations: { readOnlyHint: true } + })) + ) + const result = await run('search_mcp_tools', { query: 'issue' }) + + expect(result.matches.map((m: any) => m.tool)).toEqual(['issue_a', 'issue_b', 'issue_c']) + expect(result.matches[0].inputSchema).toBeDefined() + expect(result.matches[2].inputSchema).toBeUndefined() + expect(result.truncated).toBe(true) + }) + + // Truncating must not swallow the count of matches the score cut off: those + // tools are not in the result at all, so the model has to know to ask again. + it('keeps the more-matches hint when truncation strips schemas', async () => { + getMcpToolsMock.mockResolvedValue( + Array.from({ length: 12 }, (_, i) => ({ + name: `issue_${i}`, + description: 'An issue tool', + inputSchema: { + type: 'object', + properties: { body: { type: 'string', description: 'x'.repeat(3_000) } } + }, + annotations: { readOnlyHint: true } + })) + ) + const result = await run('search_mcp_tools', { query: 'issue' }) + + expect(result.matches).toHaveLength(10) + expect(result.note).toContain('2 more match(es)') + expect(result.note).toContain('inputSchema') + }) + + it('holds the cap when every server failed and nothing matched', async () => { + getMcpToolsMock.mockRejectedValue(new Error('x'.repeat(500))) + const servers = Array.from({ length: 50 }, (_, i) => ({ path: `u/hugo/mcp_${i}` })) + const raw = await createMcpTools(servers) + .find((t) => t.def.function.name === 'search_mcp_tools')! + .fn({ + args: { query: 'issue' }, + workspace: 'test-ws', + helpers: {}, + toolCallbacks: createToolCallbacks(), + toolId: 'tool-1' + }) + + expect(raw.length).toBeLessThanOrEqual(20_000) + expect(JSON.parse(raw).unavailableCount).toBe(50) + }) + it('truncates an oversized tools/list failure in search', async () => { getMcpToolsMock.mockRejectedValue(new Error('x'.repeat(80_000))) const result = await run('search_mcp_tools', { query: 'issue' }) diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts index d5f16d5c54..c1db1cac1b 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts @@ -2,14 +2,14 @@ import { z } from 'zod' import { ResourceService, type GetMcpToolsResponse } from '$lib/gen' import { createToolDef, type Tool } from '../shared' import { enabledMcpPaths } from '$lib/components/mcp/enabledServers' +import { isOwnOrSharedMcpPath, MCP_LIST_PER_PAGE, mcpViewer } from '$lib/components/mcp/ownServers' /** * Access to the MCP servers the user has connected (resources of type `mcp`) * as three static tools — search, read call, write call — instead of one * registered tool per remote tool. A server like GitHub's exposes ~90 tools, * whose schemas would otherwise be re-sent on every chat iteration; here only - * matched summaries enter the model's context, and a full input schema only - * after a call fails. + * the tools a search matched enter the model's context, with their schemas. */ type McpToolDef = GetMcpToolsResponse[number] @@ -17,7 +17,7 @@ type McpToolDef = GetMcpToolsResponse[number] export type McpServer = { path: string; editedAt?: string } const MAX_SEARCH_RESULTS = 10 -const MAX_DESCRIPTION_CHARS = 200 +const MAX_DESCRIPTION_CHARS = 1_000 const MAX_RESULT_CHARS = 20_000 // A server writes its own error text, and every enabled server can contribute // one, so search results are capped the same way call results are. @@ -92,13 +92,18 @@ export async function loadMcpServers(workspace: string): Promise { const enabled = enabledMcpPaths(workspace) if (enabled.length === 0) return [] try { - const resources = await ResourceService.listResource({ - workspace, - resourceType: 'mcp', - perPage: 100 - }) + const [resources, viewer] = await Promise.all([ + ResourceService.listResource({ + workspace, + resourceType: 'mcp', + perPage: MCP_LIST_PER_PAGE + }), + mcpViewer(workspace) + ]) return resources - .filter((r) => enabled.includes(r.path)) + .filter( + (r) => enabled.includes(r.path) && isOwnOrSharedMcpPath(r.path, r.extra_perms, viewer) + ) .map((r) => ({ path: r.path, editedAt: r.edited_at })) } catch (e) { console.error('Failed to load MCP servers', e) @@ -139,24 +144,17 @@ function isReadOnly(tool: McpToolDef): boolean { return tool.annotations?.readOnlyHint === true } -function schemaPropertyNames(schema: unknown): string[] { - const properties = (schema as { properties?: Record } | null | undefined) - ?.properties - return properties ? Object.keys(properties) : [] -} - function truncate(text: string, max: number): string { return text.length > max ? text.slice(0, max) + '…' : text } function summarizeTool(server: McpServer, tool: McpToolDef) { - const params = schemaPropertyNames(tool.inputSchema) return { server: server.path, tool: tool.name, description: truncate(tool.description ?? '', MAX_DESCRIPTION_CHARS), mode: isReadOnly(tool) ? 'read' : 'write', - ...(params.length > 0 ? { params } : {}) + inputSchema: tool.inputSchema } } @@ -285,28 +283,44 @@ function bounded(payload: { } /** - * The search payload under the same ceiling as a call result. Server error text - * goes first: the matches are what the model asked for. + * The search payload under the same ceiling as a call result. Schemas go before + * matches: a tool listed without its schema is still callable, since a rejected + * call returns the schema, while a tool dropped from the list is not. */ function boundedSearch(payload: { matches: unknown[]; [k: string]: unknown }): string { - const { unavailable, ...rest } = payload - const dropped = Array.isArray(unavailable) ? { unavailableCount: unavailable.length } : {} - let out = JSON.stringify(payload, null, 2) + let out = JSON.stringify(payload) if (out.length <= MAX_RESULT_CHARS) return out - const matches = [...payload.matches] - const build = () => - JSON.stringify( - { - ...rest, - ...dropped, - matches, - truncated: true, - note: `Truncated to ${MAX_RESULT_CHARS} characters. Refine the query.` - }, - null, - 2 - ) + + // The caller's note says how many matches the score cut off, which truncating + // only adds to. + const { unavailable, note: scoreNote, ...rest } = payload + const dropped = Array.isArray(unavailable) ? { unavailableCount: unavailable.length } : {} + const found = payload.matches.length + const matches = payload.matches.map((m) => ({ ...(m as object) })) as Record[] + const build = () => { + const schemaless = matches.filter((m) => !m.inputSchema).length + const note = typeof scoreNote === 'string' ? [scoreNote] : [] + note.push(`Truncated to ${MAX_RESULT_CHARS} characters.`) + if (schemaless > 0) { + note.push( + `${schemaless} lower-ranked match(es) are listed without their inputSchema; calling one returns the schema if the arguments are wrong.` + ) + } + if (matches.length < found) { + note.push(`${found - matches.length} match(es) dropped — refine the query to see them.`) + } + return JSON.stringify({ ...rest, ...dropped, matches, truncated: true, note: note.join(' ') }) + } + // Dropping the server error text is the first reduction, and the only one left + // when nothing matched. out = build() + if (out.length <= MAX_RESULT_CHARS) return out + // `matches` is ordered by score, so the tail is what the query matched least. + for (let i = matches.length - 1; i >= 0; i--) { + delete matches[i].inputSchema + out = build() + if (out.length <= MAX_RESULT_CHARS) return out + } while (out.length > MAX_RESULT_CHARS && matches.length > 0) { matches.pop() out = build() @@ -353,6 +367,14 @@ function createCallTool(servers: McpServer[], mode: 'read' | 'write'): Tool<{}> }), fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = callMcpToolSchema.parse(args) + // Marks the row with the provider it reaches. Recorded before the listing is + // awaited, so an unreachable server still marks its own failure row, and taken + // from the connected list rather than from `parsed.server`, which is a path the + // model could point anywhere. + const named = servers.find((s) => s.path === parsed.server) + if (named) { + toolCallbacks.setToolStatus(toolId, { mcpServer: { workspace, path: named.path } }) + } // Listing is a live call to a third party: a server that has gone away // must fail this tool, not the chat loop around it. let resolved: Awaited> @@ -405,7 +427,7 @@ export function createMcpTools(servers: McpServer[]): Tool<{}>[] { def: createToolDef( searchMcpToolsSchema, 'search_mcp_tools', - 'Search the tools exposed by the MCP servers connected to this workspace (listed in the system prompt). Returns server + tool names to pass to call_mcp_read_tool or call_mcp_write_tool.' + 'Search the tools exposed by the MCP servers connected to this workspace (listed in the system prompt). Returns server + tool names to pass to call_mcp_read_tool or call_mcp_write_tool, each with the input schema its arguments must follow.' ), fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = searchMcpToolsSchema.parse(args) diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index f2c3f02431..7593546114 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -141,6 +141,7 @@ function scriptDraftToWorkspaceItem(path: string, draft: NewScript): WorkspaceIt summary: draft.summary, language: draft.language, value: draft.content, + schema: draft.schema, parentHash: draft.parent_hash, isDraft: true } diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts index 7583c1b356..f696b1c869 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -338,8 +338,8 @@ export function getPipelinePromptSection(ctx: PipelineContext): string { Data Pipeline editor (ACTIVE): - The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. - Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). -- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`, \`// measure = [where ]\`, \`// dimension = \`. -- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
Triggers to deploy
+
+
+ + {#if trigger.isPrimary} + + {/if} +
+ +
+ +
+
+ 		+ {#if permission === 'deploy'} +
+ toggleTrigger(trigger, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission === 'admin-only'} + Admin only + {:else if permission === 'invalid-config'} + Invalid config + {/if} +
Agents to deploy
+
+ +
+
+ + + {agent.path} + + {#if agent.noDeployed} + Never deployed + {/if} +
+ {#if agent.noDeployed && !isSelectedAgent} + + + Never deployed, so the flow will not run until this agent is deployed. + + {/if} +
+
+ 		+ {#if permission.state === 'deploy'} +
+ toggleAgent(agent, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission.state === 'read-only'} + + Read-only + + {:else} + Invalid config + {/if} +
Triggers to deploy
-
-
- - {#if trigger.isPrimary} - - {/if} -
-
- -
-
- 		- {#if permission === 'deploy'} -
- toggleTrigger(trigger, e.detail)} - > - {#snippet children({ item })} - - - {/snippet} - -
- {:else if permission === 'admin-only'} - Admin only - {:else if permission === 'invalid-config'} - Invalid config - {/if} -
- No draft triggers found -
\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` target), \`// measure = [where ]\`, \`// dimension = \`. +- \`materialize\` (the managed output): a managed \`// materialize ducklake:///
\` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. The \`dbt://\` target below is the opposite: the node writes its own DDL and none of the write strategies apply to it. IMPORTANT: a MANAGED \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects a \`ducklake://\` target on any other language. For a \`python3\`/\`bun\`/\`postgresql\` node writing the lake, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. The one target any language BUT DBT'S OWN may declare (a dbt project's writes come from its manifest, so \`// materialize\` on a dbt script is rejected at deploy) is a WAREHOUSE RELATION: \`// materialize manual dbt:////\`, with \`\` a warehouse the workspace configures under Settings → dbt. \`manual\` is its only mode — nothing generates warehouse DDL, so the node issues its own write and the annotation records the outcome. Use it on an ingestion node a dbt project reads as a \`source\`: the declared relation and the dbt model become ONE graph node, and a downstream \`// on dbt:////\` fires when that node completes. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". - \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it. - Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. - Every node of this pipeline lives at \`f/${ctx.folder}/\` — \`${ctx.folder}\` is the folder name and \`f/\` is the owner prefix every workspace path carries, so write it exactly once (never \`f/f/…\`, and never a bare \`\`). diff --git a/frontend/src/lib/components/copilot/chat/planModeMessages.ts b/frontend/src/lib/components/copilot/chat/planModeMessages.ts index 0b502ba839..85710d0c54 100644 --- a/frontend/src/lib/components/copilot/chat/planModeMessages.ts +++ b/frontend/src/lib/components/copilot/chat/planModeMessages.ts @@ -12,6 +12,9 @@ export const PLAN_MODE_MESSAGES = { /** Sits beside the autonomy picker while plan mode holds. The picker's tooltip carries * the rest, so this states only the constraint. */ modeNote: 'Read-only', + /** Refuses a pending run form. Its own string because nothing is settled: the form stays + * live, so this names the way out rather than telling the user their run was blocked. */ + runFormRefused: 'Plan mode is read-only — switch it off to run this script.', // One pair for both artifact tools: the fact and the way forward are the same whether the // model tried to mint the plan or to rewrite it, and the generic refusal above ("put this // change in your plan") reads as nonsense for a call that writes a document. diff --git a/frontend/src/lib/components/copilot/chat/safeHref.test.ts b/frontend/src/lib/components/copilot/chat/safeHref.test.ts new file mode 100644 index 0000000000..5204ebe994 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { safeHref } from './safeHref' + +const BASE = 'https://app.example.com/sessions?workspace=demo' + +describe('safeHref', () => { + it.each([ + 'https://windmill.dev/docs', + 'http://localhost:3000/', + 'mailto:someone@example.com', + '/runs/abc', + '#anchor', + 'docs/page' + ])('keeps %s', (href) => { + expect(safeHref(href, BASE)).toBe(href) + }) + + it.each([ + 'javascript:alert(1)', + 'JavaScript:alert(1)', + ' javascript:alert(1)', + 'data:text/html,', + 'vbscript:msgbox', + 'file:///etc/passwd' + ])('drops %s', (href) => { + expect(safeHref(href, BASE)).toBeUndefined() + }) + + it('drops a missing or empty href', () => { + expect(safeHref(undefined, BASE)).toBeUndefined() + expect(safeHref('', BASE)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/safeHref.ts b/frontend/src/lib/components/copilot/chat/safeHref.ts new file mode 100644 index 0000000000..3acdabb464 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.ts @@ -0,0 +1,17 @@ +const SAFE_PROTOCOLS = ['http:', 'https:', 'mailto:'] + +/** + * The href a rendered markdown link may carry, or undefined for one that must be dropped. + * + * Markdown reaches the chat renderers from a model, and from another member for a shared + * artifact, and `svelte-exmarkdown` passes `javascript:` and `data:` hrefs through untouched. + * Relative links resolve against `base` (the page), so they stay. + */ +export function safeHref(href: string | undefined, base: string): string | undefined { + if (!href) return undefined + try { + return SAFE_PROTOCOLS.includes(new URL(href, base).protocol) ? href : undefined + } catch { + return undefined + } +} diff --git a/frontend/src/lib/components/copilot/chat/scrollFades.svelte.ts b/frontend/src/lib/components/copilot/chat/scrollFades.svelte.ts new file mode 100644 index 0000000000..0deeae535d --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/scrollFades.svelte.ts @@ -0,0 +1,56 @@ +/** + * Live "is there more past this edge" for one scroll region. Measured, not assumed: these + * boxes change height under a still scroll offset — a tool result streams in, a dynamic + * field fills its options. Put `container` on the scrolling element with + * `onscroll={measure}`, and `content` on the element inside it whose height moves. + */ +export function scrollFades() { + let node: HTMLElement | undefined = undefined + let top = $state(false) + let bottom = $state(false) + // Built on first attach, never at call time: this runs during component init, where + // ResizeObserver does not exist on the server. + let observer: ResizeObserver | undefined = undefined + + function measure() { + if (!node) { + top = false + bottom = false + return + } + top = node.scrollTop > 1 + bottom = node.scrollHeight - node.scrollTop - node.clientHeight > 1 + } + + function observe(el: HTMLElement) { + observer ??= new ResizeObserver(measure) + observer.observe(el) + return { + destroy() { + observer?.unobserve(el) + } + } + } + + return { + get top() { + return top + }, + get bottom() { + return bottom + }, + measure, + container(el: HTMLElement) { + node = el + measure() + const handle = observe(el) + return { + destroy() { + handle.destroy() + node = undefined + } + } + }, + content: observe + } +} diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index c734691e29..768b6d1b65 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -38,7 +38,7 @@ vi.mock('$lib/components/flows/flowTree', () => ({ vi.mock('$lib/gen', () => ({ ScriptService: {}, FlowService: {}, - JobService: { getJob: vi.fn() }, + JobService: { getJob: vi.fn(), getJobUpdates: vi.fn() }, ScheduleService: { previewSchedule: vi.fn(), createSchedule: vi.fn() @@ -945,6 +945,16 @@ describe('processToolCall', () => { { requestConfirmation: vi.fn().mockResolvedValue(false) } ) ).toEqual([['ai_chat', 'tool', 'run_script:declined']]) + // A tool that runs its own consent surface (the run form) declines by settling + // the card, then returns normally — that must not read as a successful run. + expect( + await outcomeKeys({ + fn: vi.fn(async ({ toolCallbacks, toolId }: any) => { + toolCallbacks.setToolStatus(toolId, { declinedByUser: true }) + return 'cancelled' + }) + }) + ).toEqual([['ai_chat', 'tool', 'run_script:declined']]) expect(await outcomeKeys({}, { isPlanModeActive: () => true })).toEqual([ ['ai_chat', 'tool', 'run_script:blocked_plan_mode'] ]) @@ -1272,6 +1282,28 @@ describe('isActiveUserQuestion', () => { }) }) +describe('isActiveRunForm', () => { + const runForm = { path: 'f/a/b', schema: {}, args: { name: 'ada' } } + function toolMessage(overrides: Partial = {}): ToolDisplayMessage { + return { + role: 'tool', + tool_call_id: 'call_r', + content: 'waiting for arguments', + isLoading: true, + runForm, + ...overrides + } + } + + // Either flag unmounts the card, so the loop must stop waiting on it. + it('is false once submitted or cancelled', async () => { + const { isActiveRunForm } = await import('./shared') + expect(isActiveRunForm(toolMessage())).toBe(true) + expect(isActiveRunForm(toolMessage({ runForm: { ...runForm, submitted: true } }))).toBe(false) + expect(isActiveRunForm(toolMessage({ runForm: { ...runForm, canceled: true } }))).toBe(false) + }) +}) + describe('pendingUserAction', () => { const toolMessage = (overrides: Partial = {}): ToolDisplayMessage => ({ role: 'tool', @@ -1289,6 +1321,15 @@ describe('pendingUserAction', () => { expect(pendingUserAction([toolMessage({ needsConfirmation: true })])).toBe('confirmation') }) + // A run form asks for arguments rather than a yes/no, but it blocks the loop the + // same way, so the chat reports it as a confirmation. + it('reports an unsubmitted run form as a confirmation', async () => { + const { pendingUserAction } = await import('./shared') + expect( + pendingUserAction([toolMessage({ runForm: { path: 'f/a/b', schema: {}, args: {} } })]) + ).toBe('confirmation') + }) + it('is undefined for a tool the AI is running on its own', async () => { const { pendingUserAction } = await import('./shared') expect(pendingUserAction([toolMessage()])).toBe(undefined) @@ -1365,6 +1406,9 @@ describe('pollJobCompletion detach', () => { const getJob = vi.mocked(JobService.getJob) getJob.mockReset() getJob.mockResolvedValue({ type: 'QueuedJob', running: true } as any) + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ running: true, completed: false } as any) const cbs = makeCallbacks() // detachAfterMs 2000 → 2 polls at 1s each, then detach. @@ -1390,14 +1434,50 @@ describe('pollJobCompletion detach', () => { const { JobService } = await import('$lib/gen') const getJob = vi.mocked(JobService.getJob) getJob.mockReset() - const completed = { type: 'CompletedJob', success: true, result: 42 } + const completed = { type: 'CompletedJob', success: true, result: 42, logs: 'ran' } getJob.mockResolvedValue(completed as any) + // Still landing on a tick the updates endpoint calls unfinished: the job can + // complete between the two calls, and `getJob` is what says so. + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ completed: false, running: true } as any) const cbs = makeCallbacks() const promise = pollJobCompletion('job1', 'w', 'tool1', cbs as any, { detachAfterMs: 15000 }) await vi.advanceTimersByTimeAsync(1000) - expect(await promise).toBe(completed) + const landed = await promise + expect(landed).toBe(completed) + // Fetched again with its logs rather than settled on the logless tick fetch, + // which would reach the model as "No logs available". + expect((landed as any).logs).toBe('ran') + } finally { + vi.useRealTimers() + } + }) + + // Streaming rides on a second endpoint; landing the job must not. A failing updates + // endpoint costs live logs, never the run. + it('returns the completed job with its logs when the updates endpoint fails', async () => { + vi.useFakeTimers() + try { + const { pollJobCompletion } = await import('./shared') + const { JobService } = await import('$lib/gen') + const getJob = vi.mocked(JobService.getJob) + getJob.mockReset() + const completed = { type: 'CompletedJob', success: true, result: 42, logs: 'ran' } + getJob.mockResolvedValue(completed as any) + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockRejectedValue(new Error('updates unavailable')) + const cbs = makeCallbacks() + + const promise = pollJobCompletion('job1', 'w', 'tool1', cbs as any, { detachAfterMs: 15000 }) + await vi.advanceTimersByTimeAsync(1000) + + const landed = await promise + expect(landed).toBe(completed) + expect((landed as any).logs).toBe('ran') } finally { vi.useRealTimers() } @@ -1411,6 +1491,9 @@ describe('pollJobCompletion detach', () => { const getJob = vi.mocked(JobService.getJob) getJob.mockReset() getJob.mockResolvedValue({ type: 'QueuedJob', running: true } as any) + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ running: true, completed: false } as any) const cbs = makeCallbacks() const promise = pollJobCompletion('job1', 'w', 'tool1', cbs as any) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 6d6794b56c..73efd2e37e 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -553,6 +553,57 @@ export function answeredChoices(q: UserQuestionDisplay): string[] | undefined { return q.selectedChoices ?? (q.selectedChoice ? [q.selectedChoice] : undefined) } +/** Argument form for a deployed-script run, persisted with the transcript — every + * field has to stay plain JSON. */ +export type RunFormDisplay = { + path: string + summary?: string + /** What the run is, in the card's own words: a deployed script run, or a preview of the + * draft being written. Only the tense of the row's label turns on it. */ + kind?: 'run' | 'test' + /** What is being run, for the noun the card says it in. + * Absent on cards recorded before flows had a form, which were all scripts. */ + runnableKind?: 'script' | 'flow' + /** Of whatever version is about to run: the deployed script, or the draft a test + * previews. Only the rendered form reads it, so it is dropped once one of the flags + * below unmounts that form: kept, every settled card would carry a copy of the schema + * — password and file defaults included — in history forever. */ + schema?: Record + /** The script the `dynselect-` helper runs: the draft a script test run previews, since a + * deployed helper would answer for the wrong version, or the one a flow schema carries on + * itself. Dropped with the schema once the form unmounts, so no settled card keeps a copy + * of the code. */ + code?: string + lang?: ScriptLang + /** Prefill only: the card's `parameters` records what the job started with. */ + args: Record + /** Proposed arguments emptied because their declared type had no reading of them. + * Named on the card: an empty field is otherwise the caller having sent nothing. */ + clearedKeys?: string[] + /** Proposed arguments a disabled field overrode with its default. Named for the same + * reason: the field renders locked, so the value it holds is not the proposed one. */ + resetKeys?: string[] + /** File arguments emptied out of the proposal. Named so an empty field reads as the + * caller's value having been removed, not as the field having none. */ + strippedKeys?: string[] + /** Either one unmounts the form, so set exactly one, and only once the loop has + * stopped waiting on this card. */ + submitted?: boolean + canceled?: boolean + /** The job exists. Distinct from `submitted`, which flips a round trip earlier — in + * between, whether the server queued a job is unknown, so a turn stopped there is + * recorded as neither started nor canceled. */ + started?: boolean +} + +/** What a run form is being filled with while it waits. Held by the chat manager, not by + * the form, so the chat card and the preview pane edit one draft rather than two copies. + * The schema rides along because SchemaForm binds and reorders it. */ +export type RunFormDraft = { + args: Record + schema: Record +} + /** One page hit from a provider-side web search (OpenAI sources carry no title). */ export type WebSearchSource = { url: string @@ -565,6 +616,9 @@ export type ToolDisplayMessage = { content: string parameters?: any result?: any + /** What the job has streamed of its result so far, while it is still running. + * Cleared when the job lands: `result` is then the whole of it. */ + resultStream?: string logs?: string isLoading?: boolean /** Arguments fully streamed but execution not started (see queuedToolStatus). */ @@ -575,9 +629,15 @@ export type ToolDisplayMessage = { autoCollapseDetails?: boolean isStreamingArguments?: boolean toolName?: string + /** What marks this row with its provider. Recorded rather than looked up, because the + * server listing lives only in memory and a reloaded transcript does not have it. The + * workspace rides along: a chat is readable from any workspace, and the same path names + * a different server in each. */ + mcpServer?: { workspace: string; path: string } showFade?: boolean actions?: ToolDisplayAction[] userQuestion?: UserQuestionDisplay + runForm?: RunFormDisplay webSearchSources?: WebSearchSource[] /** Data URL of an image the tool produced (e.g. take_screenshot), shown on the card. */ imageUrl?: string @@ -653,6 +713,18 @@ export function isActiveUserQuestion(message: DisplayMessage | undefined): boole ) } +export function isActiveRunForm(message: DisplayMessage | undefined): boolean { + return Boolean( + message && + message.role === 'tool' && + message.runForm && + message.isLoading && + !message.error && + !message.runForm.submitted && + !message.runForm.canceled + ) +} + // The loop is parked on the user: an unanswered askUserQuestion, or a tool call // staged for confirmation. The manager stays `loading` through both, so anything // rendering progress must ask here first or it reports "the AI is working". @@ -676,6 +748,12 @@ export function pendingUserActionDetail( if (isActiveUserQuestion(message)) { return { action: 'question', toolCallId: message.tool_call_id } } + // A run form is a confirmation carrying arguments, not a question: it parks the + // turn the same way, but Run or Cancel resolves it and typing never does — so it + // must not claim the answer affordance a pending question offers. + if (isActiveRunForm(message)) { + return { action: 'confirmation', toolCallId: message.tool_call_id } + } if (message.needsConfirmation && message.isLoading) { return { action: 'confirmation', toolCallId: message.tool_call_id } } @@ -958,6 +1036,9 @@ export async function processToolCall({ } let result = '' + // A tool that asks for consent itself settles the call as declined or blocked and + // returns normally, so without this both telemeter as successful runs. + let settledInsideTool: 'declined' | 'blocked_plan_mode' | undefined = undefined try { result = await callTool({ tools, @@ -965,10 +1046,17 @@ export async function processToolCall({ args, workspace: workspaceId, helpers, - toolCallbacks, + toolCallbacks: { + ...toolCallbacks, + setToolStatus: (toolId, status) => { + if (status?.declinedByUser) settledInsideTool = 'declined' + else if (status?.blockedByPlanMode) settledInsideTool = 'blocked_plan_mode' + toolCallbacks.setToolStatus(toolId, status) + } + }, toolId: toolCall.id }) - logToolOutcome('ok') + logToolOutcome(settledInsideTool ?? 'ok') toolCallbacks.setToolStatus(toolCall.id, { isLoading: false, isStreamingArguments: false @@ -1069,6 +1157,10 @@ export interface Tool { * is true. */ refuseInPlanMode?: (p: { args: any; helpers: T }) => ToolRejection | undefined requiresConfirmation?: boolean + /** The tool's own argument form is its confirmation, and the bypass posture answers that + * form — so no card is waited on, yet a decision is still being made for the user. The + * list of what the posture bypasses is built from both this and `requiresConfirmation`. */ + bypassedByAutoAccept?: boolean /** Header shown on the confirmation card before the tool runs. Pass a function * to derive it from the parsed arguments (e.g. name the script being tested). */ confirmationMessage?: string | ((args: any) => string) @@ -1229,6 +1321,16 @@ export interface ToolCallbacks { toolId: string, question: UserQuestionDisplay ) => Promise + /** Park the loop on an argument form and resolve with the args the user submitted, or + * undefined if they cancelled. Wired only where the form can be rendered. `autoAccepted` + * says YOLO already answered it with what it opened with, so there is no card to wait on. */ + requestRunArgs?: ( + toolId: string, + form: RunFormDisplay, + opts?: { autoAccepted?: boolean } + ) => Promise | undefined> + /** The submitted form's job is queued. Wired alongside requestRunArgs. */ + markRunFormStarted?: (toolId: string) => void /** Records a workspace item the tool call created/edited/deleted, by its * canonical (itemKind, storagePath). Session chats wire this to accumulate the * chat's modified-items mask; the global side-panel chat omits it (no-op). */ @@ -1488,7 +1590,7 @@ export async function buildSchemaForTool( // Constants for result formatting const MAX_RESULT_LENGTH = 12000 -const MAX_LOG_LENGTH = 4000 +export const MAX_LOG_LENGTH = 4000 export const MAX_RUNNABLE_CONTENT_LENGTH = 20000 /** How long a test run is awaited inline before it detaches into the background @@ -1517,13 +1619,16 @@ export interface TestRunConfig { detachAfterMs?: number /** Human label for the jobs tray row (path / step id). Defaults to the job id. */ label?: string - /** Overrides the default "…test started, waiting for completion" status while the + /** Overrides the default "…started, waiting for completion" status while the * job runs inline (e.g. an SQL tool shows "SQL running…"). */ runningMessage?: string - /** Noun for the human-facing status strings (" test completed successfully"). - * Defaults to `contextName`, which also carries the jobs-tray kind and so cannot - * always name what ran: an app's path runnable queues a flow job. */ + /** The item noun in the human-facing status strings ("Flow test completed + * successfully"). Defaults to `contextName`, which also carries the jobs-tray kind + * and so cannot always name what ran: an app's path runnable queues a flow job. */ completionName?: string + /** The action noun in those same strings ("Script run completed successfully"). + * Defaults to "test", so a tool running the deployed item for real passes "run". */ + actionNoun?: string /** Custom terminal formatting for the INLINE completion path (callers whose * result isn't a plain test-run summary, e.g. exec_datatable_sql shaping rows). * Returns the string handed to the model plus the tool-card patch. When omitted, @@ -1544,6 +1649,46 @@ export type BackgroundJobFormatter = (job: CompletedJob) => { card: Partial } +/** Reads a running job's output incrementally through `getJobUpdates`, the only endpoint + * carrying `new_result_stream` — `getJob` returns logs but never the partial result. Each + * reader keeps its own offsets, so one starting over refetches from zero. Best-effort: a + * failed poll answers `undefined` and mutates nothing, so the next resumes from the same + * offsets and a run always lands on `getJob` alone. */ +export function createJobUpdateReader(jobId: string, workspace: string) { + let logs = '' + let resultStream = '' + let logOffset = 0 + let streamOffset = 0 + let started = false + return { + async poll(): Promise<{ completed: boolean; logs: string; resultStream: string } | undefined> { + let update: Awaited> + try { + update = await JobService.getJobUpdates({ + workspace, + id: jobId, + running: started, + logOffset, + streamOffset + }) + } catch { + return undefined + } + started ||= update.running ?? false + // Both kept as a tail: the offsets come from the server, so dropping the head + // costs nothing here, and neither is the record of the run — the logs are on the + // job, and a streamed partial is replaced by the result the moment it lands. + if (update.new_logs) logs = (logs + update.new_logs).slice(-MAX_LOG_LENGTH) + if (update.new_result_stream) { + resultStream = (resultStream + update.new_result_stream).slice(-MAX_LOG_LENGTH) + } + if (update.log_offset) logOffset = update.log_offset + if (update.stream_offset) streamOffset = update.stream_offset + return { completed: update.completed ?? false, logs, resultStream } + } + } +} + // Common job polling function. // // Two modes, selected by whether `detachAfterMs` is provided: @@ -1563,24 +1708,54 @@ export async function pollJobCompletion( const maxAttempts = detachEnabled ? Math.ceil((options?.detachAfterMs ?? 0) / 1000) : 60 let attempts = 0 let job: CompletedJob | null = null + const reader = createJobUpdateReader(jobId, workspace) while (attempts < maxAttempts) { await new Promise((resolve) => setTimeout(resolve, 1000)) attempts++ try { + const update = await reader.poll() + // The tray's snapshot is trimmed of logs (it is persisted), so the card is the + // only place a running job's output can land. Cards that hide their logs while + // loading are unaffected; the run card follows them line by line. + if (update) { + toolCallbacks.setToolStatus(toolId, { + logs: formatLogs(update.logs), + resultStream: update.resultStream || undefined + }) + } + + // Ask for the logs when the run may be over — the tail written between the last + // poll and the end is only on the job itself — or when there is no reader output + // to have collected them. + const wantLogs = !update || update.completed const fetchedJob = await JobService.getJob({ workspace: workspace, id: jobId, - noLogs: false, + noLogs: !wantLogs, noCode: true }) - if (fetchedJob.type === 'CompletedJob') { - job = fetchedJob + // The updates can still call a landed job unfinished, so a completion seen on + // a logless fetch is fetched again rather than settled without them: the model + // reads these logs, and their absence is indistinguishable from a silent run. + job = wantLogs + ? fetchedJob + : ((await JobService.getJob({ + workspace: workspace, + id: jobId, + noLogs: false, + noCode: true + })) as CompletedJob) break } - // Keep the tray's status + Job snapshot fresh during the inline wait. + // With no reader, this is the only place the card's logs can come from. + if (!update) { + toolCallbacks.setToolStatus(toolId, { logs: formatLogs(fetchedJob.logs) }) + } + // The badge needs the real Job to tell running from suspended or scheduled, which + // the updates do not say. toolCallbacks.onJobStatus?.(jobId, { status: deriveChatJobStatus(fetchedJob), job: trimJob(fetchedJob) @@ -1694,14 +1869,22 @@ function backgroundedSummary(jobId: string, label: string): string { // fills its card the same way one that finished inline does. export function completedJobToolStatus(job: CompletedJob): Partial { // A canceled job isn't a `success`, but it isn't a failure either — the user - // stopped it — so don't dress the card as an error. + // stopped it — so don't dress the card as an error. It still has the result the run + // page shows for a canceled run, which names who stopped it, so keep that. if (job.canceled) { - return { content: 'Background job canceled', logs: formatLogs(job.logs) } + return { + content: 'Background job canceled', + result: formatResult(job.result), + logs: formatLogs(job.logs), + resultStream: undefined + } } return { content: `Background job ${job.success ? 'completed successfully' : 'failed'}`, result: formatResult(job.result), logs: formatLogs(job.logs), + // The partial is the result now, so nothing streamed is kept beside it. + resultStream: undefined, ...(job.success ? {} : { error: getErrorMessage(job.result) }) } } @@ -1731,12 +1914,15 @@ export function backgroundJobCompletionNote( ) } -// Main execution function for test runs export async function executeTestRun(config: TestRunConfig): Promise { // Detach-into-background is enabled only when the host wired the job hooks // (global/sessions chat). Otherwise this stays a blocking call. const detachEnabled = !!config.toolCallbacks.onJobStarted const label = config.label ?? config.contextName + const actionNoun = config.actionNoun ?? 'test' + // Stands on its own where the status strings are prefixed by the item, so its + // default carries the noun. + const failureNoun = config.actionNoun ?? 'test run' try { config.toolCallbacks.setToolStatus(config.toolId, { content: config.startMessage || `Starting ${config.contextName} test...` @@ -1761,7 +1947,8 @@ export async function executeTestRun(config: TestRunConfig): Promise { }) config.toolCallbacks.setToolStatus(config.toolId, { - content: config.runningMessage ?? `${contextName} test started, waiting for completion...` + content: + config.runningMessage ?? `${contextName} ${actionNoun} started, waiting for completion...` }) const outcome = await pollJobCompletion( @@ -1781,7 +1968,7 @@ export async function executeTestRun(config: TestRunConfig): Promise { if (outcome === 'detached') { config.toolCallbacks.onJobDetached?.(jobId) config.toolCallbacks.setToolStatus(config.toolId, { - content: `${contextName} test running in background (job ${jobId})` + content: `${contextName} ${actionNoun} running in background (job ${jobId})` }) return backgroundedSummary(jobId, label) } @@ -1800,9 +1987,12 @@ export async function executeTestRun(config: TestRunConfig): Promise { } config.toolCallbacks.setToolStatus(config.toolId, { - content: `${contextName} test ${job.success ? 'completed successfully' : 'failed'}`, + content: `${contextName} ${actionNoun} ${job.success ? 'completed successfully' : 'failed'}`, result: formatResult(job.result), logs: formatLogs(job.logs), + // The partial is the result now, so the card reads it off `result` alone and the + // transcript stops carrying a second copy of a streamed answer. + resultStream: undefined, ...(job.success ? {} : { error: getErrorMessage(job.result) }) }) @@ -1823,33 +2013,60 @@ export async function executeTestRun(config: TestRunConfig): Promise { // the flow it could not find — losing the one diagnostic the run exists for. const errorMessage = formatToolError(error) config.toolCallbacks.setToolStatus(config.toolId, { - content: `Test execution failed`, + content: `Execution failed`, error: errorMessage }) - throw new Error(`Failed to execute test run: ${errorMessage}`) + throw new Error(`Failed to execute ${failureNoun}: ${errorMessage}`) } } type FlowStepScriptLoader = ( moduleValue: { path: string; hash?: string }, workspace: string -) => Promise<{ content: string; language: ScriptLang }> +) => Promise<{ content: string; language: ScriptLang; schema?: Record }> -type FlowStepPreviewLoader = (path: string, workspace: string) => Promise +/** A subflow step's target. `previewValue` is set only when a draft exists — that is what + * decides between previewing the draft and running the deployed flow by path — while + * `schema` describes whichever of the two is about to run. */ +type FlowStepSubflowLoader = ( + path: string, + workspace: string +) => Promise<{ previewValue?: FlowValue; schema?: Record } | undefined> -export type FlowStepTestRunConfig = { +type FlowStepRunConfig = { flowValue: FlowValue stepId: string - args?: Record | null workspace: string toolCallbacks: ToolCallbacks toolId: string + loadScript?: FlowStepScriptLoader + loadSubflow?: FlowStepSubflowLoader +} + +export type FlowStepTestRunConfig = FlowStepRunConfig & { + args?: Record | null background?: boolean /** Inline wait budget (ms) before the step job detaches into the tray; forwarded * to executeTestRun. Ignored when `background` is set. */ detachAfterMs?: number - loadScript?: FlowStepScriptLoader - loadFlowPreviewValue?: FlowStepPreviewLoader +} + +/** One step resolved to the job it would start, short of starting it, so a caller that + * puts an argument form in front of the run can build the form's fields from the same + * read the job uses. Schema inference lives with the caller: this module is kept on a + * shallow import list (see the note at the top of the file). */ +export type ResolvedFlowStepRun = { + module: FlowModule + runnableKind: 'script' | 'flow' + /** A subflow step carries `schema` instead, having no code of its own to read. */ + code?: string + lang?: ScriptLang + schema?: Record + startMessage: string + /** Takes the arguments as submitted. The preprocessor's entrypoint override is added + * here rather than by the caller: it is declared by no schema, so anything that + * conforms arguments to one would drop it. */ + startJob: (args: Record) => Promise } function normalizeFlowStepArgs(args: Record | null | undefined): Record { @@ -1875,25 +2092,26 @@ function getAvailableFlowStepIds(flowValue: FlowValue): string { async function loadDeployedScriptForFlowStep( moduleValue: { path: string; hash?: string }, workspace: string -): Promise<{ content: string; language: ScriptLang }> { +): Promise<{ content: string; language: ScriptLang; schema?: Record }> { const script = moduleValue.hash ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) - return { content: script.content, language: script.language } + return { + content: script.content, + language: script.language, + schema: script.schema as Record | undefined + } } -export async function executeFlowStepTestRun({ +export async function resolveFlowStepRun({ flowValue, stepId, - args, workspace, toolCallbacks, toolId, - background, - detachAfterMs, loadScript = loadDeployedScriptForFlowStep, - loadFlowPreviewValue -}: FlowStepTestRunConfig): Promise { + loadSubflow +}: FlowStepRunConfig): Promise { const targetModule = findModuleInFlow(flowValue, stepId) ?? undefined if (!targetModule) { @@ -1907,94 +2125,76 @@ export async function executeFlowStepTestRun({ } const moduleValue = targetModule.value - const stepArgs = normalizeFlowStepArgs(args) + const withEntrypoint = (args: Record) => flowStepArgsForModule(targetModule.id, args) if (moduleValue.type === 'rawscript') { - return executeTestRun({ - jobStarter: () => + return { + module: targetModule, + runnableKind: 'script', + code: moduleValue.content ?? '', + lang: moduleValue.language, + startMessage: `Starting test run of step "${stepId}"...`, + startJob: (args) => JobService.runScriptPreview({ workspace, requestBody: { content: moduleValue.content ?? '', language: moduleValue.language, - args: flowStepArgsForModule(targetModule.id, stepArgs) + args: withEntrypoint(args) } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of step "${stepId}"...`, - contextName: 'script', - label: `step ${stepId}`, - background, - detachAfterMs - }) + }) + } } if (moduleValue.type === 'script') { const script = await loadScript(moduleValue, workspace) - return executeTestRun({ - jobStarter: () => + return { + module: targetModule, + runnableKind: 'script', + code: script.content, + lang: script.language, + schema: script.schema, + startMessage: `Starting test run of script step "${stepId}"...`, + startJob: (args) => JobService.runScriptPreview({ workspace, requestBody: { path: moduleValue.path, content: script.content, language: script.language, - args: flowStepArgsForModule(targetModule.id, stepArgs) + args: withEntrypoint(args) } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of script step "${stepId}"...`, - contextName: 'script', - label: `step ${stepId}`, - background, - detachAfterMs - }) + }) + } } if (moduleValue.type === 'flow') { - const previewValue = await loadFlowPreviewValue?.(moduleValue.path, workspace) - if (previewValue) { - return executeTestRun({ - jobStarter: () => - JobService.runFlowPreview({ - workspace, - requestBody: { + const subflow = await loadSubflow?.(moduleValue.path, workspace) + const previewValue = subflow?.previewValue + return { + module: targetModule, + runnableKind: 'flow', + schema: subflow?.schema, + startMessage: previewValue + ? `Starting test run of draft flow step "${stepId}"...` + : `Starting test run of flow step "${stepId}"...`, + startJob: (args) => + previewValue + ? JobService.runFlowPreview({ + workspace, + requestBody: { path: moduleValue.path, value: previewValue, args } + }) + : JobService.runFlowByPath({ + workspace, path: moduleValue.path, - value: previewValue, - args: stepArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of draft flow step "${stepId}"...`, - contextName: 'flow', - label: `step ${stepId}`, - background, - detachAfterMs - }) + requestBody: args, + // As the flow editor's own step test does: these are the subflow's main input + // schema's arguments, and a preprocessor would take them for a trigger event + // and hand the flow its own output instead. A parent flow runs a subflow step + // the same way (apply_preprocessor: false). + skipPreprocessor: true + }) } - - return executeTestRun({ - jobStarter: () => - JobService.runFlowByPath({ - workspace, - path: moduleValue.path, - requestBody: stepArgs - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of flow step "${stepId}"...`, - contextName: 'flow', - label: `step ${stepId}`, - background, - detachAfterMs - }) } toolCallbacks.setToolStatus(toolId, { @@ -2006,6 +2206,26 @@ export async function executeFlowStepTestRun({ ) } +export async function executeFlowStepTestRun({ + args, + background, + detachAfterMs, + ...config +}: FlowStepTestRunConfig): Promise { + const resolved = await resolveFlowStepRun(config) + return executeTestRun({ + jobStarter: () => resolved.startJob(normalizeFlowStepArgs(args)), + workspace: config.workspace, + toolCallbacks: config.toolCallbacks, + toolId: config.toolId, + startMessage: resolved.startMessage, + contextName: resolved.runnableKind, + label: `step ${config.stepId}`, + background, + detachAfterMs + }) +} + function formatLogs(logs: string | undefined): undefined | string { if (logs && logs.trim()) { if (logs.length <= MAX_LOG_LENGTH) { diff --git a/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts index e91d40bff0..1ac4439a9b 100644 --- a/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts +++ b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts @@ -1,10 +1,11 @@ -import { createEnabledPathsPreference } from '../enabledPathsPreference' +import { createPathsPreference } from '../enabledPathsPreference' /** Which `ai_skill` resources the chat may follow, per workspace and per account. - * Every enabled skill spends context on every turn, so selecting one is a personal - * choice rather than a consequence of being able to read it. */ -const preference = createEnabledPathsPreference('wm_skills_enabled') + * Every skill readable in the workspace is on unless someone turned it off: a skill + * is instructions the workspace wrote for the assistant to use, so what carrying one + * costs is context, not access. */ +const preference = createPathsPreference('wm_skills_enabled', true) -export const enabledSkillPaths = preference.enabledPaths export const isSkillEnabled = preference.isEnabled export const setSkillEnabled = preference.setEnabled +export const forgetSkill = preference.forget diff --git a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts index 20737caffa..7a80705d4e 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts @@ -38,14 +38,14 @@ export function ambiguousSkillNames(skills: readonly { name: string }[]): Set/…`, a folder the user owns), which is why this goes through * the shared `canWrite` rather than reading `extra_perms` alone. */ -export async function listSkillResources( - workspace: string, - user?: UserExt -): Promise { +export async function listSkillResources(workspace: string, user?: UserExt): Promise { if (!workspace) return { skills: [], truncated: false } const rows: SkillResource[] = [] for (let page = 1; page <= MAX_SKILLS_PAGES; page++) { diff --git a/frontend/src/lib/components/copilot/chat/skills/skillTree.ts b/frontend/src/lib/components/copilot/chat/skills/skillTree.ts new file mode 100644 index 0000000000..d2de4eb655 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/skillTree.ts @@ -0,0 +1,119 @@ +/** Grouping for the Skills list: skills sorted into the folders that hold them. + * + * Only worth showing when the skills are spread over more than one folder — see + * `skillFolderPaths` — since a tree over a single folder is a list with a header + * on top of it. */ + +/** A folder in the skills tree. `path` is the whole prefix the node covers, so it + * is unique across the tree and can key collapsed state. */ +export type SkillTreeNode = { + path: string + /** Header text: the whole scope (`u/admin`, `f/skills`) at the root, one + * segment (`deploy`) deeper, where the ancestors are already on screen. */ + label: string + /** A `u/`, `f/` or `g/` root rather than a subfolder. */ + scope: boolean + children: SkillTreeNode[] + skills: T[] +} + +type Skill = { path: string; name: string } + +/** The folders holding these skills — one entry per distinct parent path. */ +export function skillFolderPaths(skills: readonly Skill[]): Set { + return new Set(skills.map((s) => s.path.split('/').slice(0, -1).join('/'))) +} + +/** Every skill under `node`, its subfolders included. */ +export function countSkills(node: SkillTreeNode): number { + return node.skills.length + node.children.reduce((n, c) => n + countSkills(c), 0) +} + +/** The skills a folder switch acts on: everything under it, however deep. */ +export function nodeSkills(node: SkillTreeNode): T[] { + return [...node.skills, ...node.children.flatMap((c) => nodeSkills(c))] +} + +/** One line of the rendered list, folders included. */ +export type SkillTreeEntry = { key: string; parentKey?: string } & ( + | { kind: 'folder'; node: SkillTreeNode } + | { kind: 'skill'; skill: T } +) + +/** A folder and a skill can hold the same path — `f/skills/deploy` names both when + * `f/skills/deploy/rollback` exists beside it — so the kind is part of the key. */ +export function folderKey(path: string): string { + return `folder:${path}` +} +export function skillKey(path: string): string { + return `skill:${path}` +} + +/** The list as it stands on screen, top to bottom, with collapsed folders holding + * their contents back. This is what the keyboard walks; the markup renders the same + * forest recursively under the same `isCollapsed`, so the two agree line for line. */ +export function visibleEntries( + forest: readonly SkillTreeNode[], + isCollapsed: (path: string) => boolean +): SkillTreeEntry[] { + const out: SkillTreeEntry[] = [] + const walk = (node: SkillTreeNode, parentKey?: string) => { + const key = folderKey(node.path) + out.push({ kind: 'folder', key, parentKey, node }) + if (isCollapsed(node.path)) return + for (const child of node.children) walk(child, key) + for (const skill of node.skills) { + out.push({ kind: 'skill', key: skillKey(skill.path), parentKey: key, skill }) + } + } + for (const root of forest) walk(root) + return out +} + +/** Sort a skill's path into the forest, creating the folders it names. + * + * Paths are `[ufg]//` and may nest further (`f/skills/deploy/rollback` + * passes the resource path CHECK), so the first two segments are one node and each + * segment after that is a node of its own. */ +export function buildSkillTree(skills: readonly T[]): SkillTreeNode[] { + const roots = new Map>() + for (const skill of skills) { + const parts = skill.path.split('/') + // Two segments cannot happen through the API, but a hand-written path in a + // test or a future path shape should land somewhere rather than vanish. + const rootPath = parts.slice(0, 2).join('/') + let node: SkillTreeNode = roots.get(rootPath) ?? { + path: rootPath, + label: rootPath, + scope: true, + children: [], + skills: [] + } + roots.set(rootPath, node) + for (const segment of parts.slice(2, -1)) { + const path = `${node.path}/${segment}` + let child = node.children.find((c) => c.path === path) + if (child === undefined) { + child = { path, label: segment, scope: false, children: [], skills: [] } + node.children.push(child) + } + node = child + } + node.skills.push(skill) + } + + const sort = (node: SkillTreeNode) => { + node.children.sort((a, b) => a.label.localeCompare(b.label)) + node.skills.sort((a, b) => a.name.localeCompare(b.name)) + node.children.forEach(sort) + } + const forest = [...roots.values()] + forest.forEach(sort) + // Personal scopes before shared folders, alphabetical within each: `u/` is + // where this modal's New skill and folder import write, so it is the half of the + // list someone is most often here to change. + return forest.sort((a, b) => { + const scopeRank = (p: string) => (p.startsWith('u/') ? 0 : 1) + return scopeRank(a.path) - scopeRank(b.path) || a.path.localeCompare(b.path) + }) +} diff --git a/frontend/src/lib/components/copilot/chat/skills/skills.test.ts b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts index 5b0d2196ca..ae2e737af2 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skills.test.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts @@ -9,8 +9,10 @@ vi.mock('$lib/stores', () => ({ userStore: { subscribe: (run: (v: unknown) => void) => (run({ ...session }), () => {}) } })) -import { enabledSkillPaths, isSkillEnabled, setSkillEnabled } from './enabledSkills' +import { isMcpEnabled } from '$lib/components/mcp/enabledServers' +import { isSkillEnabled, setSkillEnabled } from './enabledSkills' import { ambiguousSkillNames, truncateChars, truncateForPrompt } from './skillResources' +import { buildSkillTree, countSkills, skillFolderPaths, visibleEntries } from './skillTree' describe('enabledSkills', () => { beforeEach(() => { @@ -18,22 +20,103 @@ describe('enabledSkills', () => { session.email = 'first@windmill.dev' }) - it('keeps the selection separate per workspace', () => { - setSkillEnabled('ws_a', 'u/me/deploy', true) - expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) - expect(isSkillEnabled('ws_b', 'u/me/deploy')).toBe(false) + it('keeps a skill turned off separate per workspace', () => { + setSkillEnabled('ws_a', 'u/me/deploy', false) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(false) + expect(isSkillEnabled('ws_b', 'u/me/deploy')).toBe(true) }) - it('does not hand the next account the previous one’s selection', () => { - setSkillEnabled('ws_a', 'u/me/deploy', true) + it('does not hand the next account the previous one’s choice', () => { + setSkillEnabled('ws_a', 'u/me/deploy', false) session.email = 'second@windmill.dev' - expect(enabledSkillPaths('ws_a')).toEqual([]) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) }) it('reports failure when there is no account to record the choice against', () => { session.email = undefined - expect(setSkillEnabled('ws_a', 'u/me/deploy', true)).toBe(false) - expect(enabledSkillPaths('ws_a')).toEqual([]) + expect(setSkillEnabled('ws_a', 'u/me/deploy', false)).toBe(false) + }) +}) + +// Storage written before skills defaulted to on holds an array of the paths that +// were on. Nothing rewrites it, so reading it wrong is what would silently move +// somebody's choices — in either direction, for either default. +describe('choices stored under the older shape', () => { + const scope = 'ws_a:first@windmill.dev' + + beforeEach(() => { + localStorage.clear() + session.email = 'first@windmill.dev' + }) + + it('keeps a skill that was turned on, and defaults the rest to on', () => { + localStorage.setItem('wm_skills_enabled', JSON.stringify({ [scope]: ['u/me/deploy'] })) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) + expect(isSkillEnabled('ws_a', 'u/me/never-picked')).toBe(true) + }) + + it('leaves an MCP server that was never turned on off', () => { + localStorage.setItem('wm_mcp_enabled', JSON.stringify({ [scope]: ['u/me/github'] })) + expect(isMcpEnabled('ws_a', 'u/me/github')).toBe(true) + expect(isMcpEnabled('ws_a', 'u/me/other')).toBe(false) + }) + + it('records an off decision beside the entries already there', () => { + localStorage.setItem('wm_skills_enabled', JSON.stringify({ [scope]: ['u/me/deploy'] })) + setSkillEnabled('ws_a', 'u/me/review', false) + expect(isSkillEnabled('ws_a', 'u/me/review')).toBe(false) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) + }) + + it('drops a path put back to the default instead of storing it', () => { + setSkillEnabled('ws_a', 'u/me/review', false) + setSkillEnabled('ws_a', 'u/me/review', true) + expect(JSON.parse(localStorage.getItem('wm_skills_enabled') ?? '{}')[scope]).toEqual({}) + }) +}) + +describe('skill tree', () => { + const skills = [ + { path: 'f/skills/deploy/rollback', name: 'rollback' }, + { path: 'f/skills/onboarding', name: 'onboarding' }, + { path: 'u/admin/release-notes', name: 'release-notes' } + ] + + it('nests each path segment under its owner, own folder first', () => { + const tree = buildSkillTree(skills) + expect(tree.map((n) => n.path)).toEqual(['u/admin', 'f/skills']) + const shared = tree[1] + expect(shared.skills.map((s) => s.name)).toEqual(['onboarding']) + // A skill two levels down gets its folder, rather than being flattened into + // the owner's own rows where the path it came from is lost. + expect(shared.children.map((c) => c.path)).toEqual(['f/skills/deploy']) + expect(shared.children[0].skills.map((s) => s.name)).toEqual(['rollback']) + expect(countSkills(shared)).toBe(2) + }) + + // The keyboard walks this list while the markup renders the forest recursively. If + // the two disagree, Down lands on a row that is not the one lit. + it('lists what is on screen, and holds back what a collapsed folder hides', () => { + const tree = buildSkillTree(skills) + expect(visibleEntries(tree, () => false).map((e) => e.key)).toEqual([ + 'folder:u/admin', + 'skill:u/admin/release-notes', + 'folder:f/skills', + 'folder:f/skills/deploy', + 'skill:f/skills/deploy/rollback', + 'skill:f/skills/onboarding' + ]) + expect(visibleEntries(tree, (path) => path === 'f/skills').map((e) => e.key)).toEqual([ + 'folder:u/admin', + 'skill:u/admin/release-notes', + 'folder:f/skills' + ]) + }) + + // What the list uses to choose between the tree and a flat list. + it('counts the folders holding the skills, not the skills', () => { + expect(skillFolderPaths(skills).size).toBe(3) + expect(skillFolderPaths([skills[1], { path: 'f/skills/deploy', name: 'deploy' }]).size).toBe(1) }) }) @@ -55,7 +138,9 @@ describe('prompt truncation', () => { const body = '漢'.repeat(100) // 300 bytes expect(truncateForPrompt(body, 3000)).toBe(body) const cut = truncateForPrompt(body, 30) - expect(new TextEncoder().encode(cut.replace('… [truncated]', '')).byteLength).toBeLessThanOrEqual(30) + expect( + new TextEncoder().encode(cut.replace('… [truncated]', '')).byteLength + ).toBeLessThanOrEqual(30) expect(cut).toContain('[truncated]') // A byte-aligned cut must not leave a broken code point behind. expect(cut).not.toContain('\ufffd') diff --git a/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts b/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts index 15f6f2839d..dd480c5e97 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts @@ -65,26 +65,26 @@ export class SkillsMenu { async #toggle(ws: string, path: string, enabled: boolean) { // A session whose fork is still staged has no workspace of its own yet, so `ws` - // is the PARENT: the selection would be stored under it and quietly stop + // is the PARENT: the choice would be stored under it and quietly stop // applying the moment the first send commits the fork. const pendingForkOf = this.#manager.sessionContextResolver?.()?.pendingForkOf if (pendingForkOf !== undefined) { sendUserToast( - `This session has not created its workspace yet, so the selection would be stored under "${pendingForkOf}". Send a message first.`, + `This session has not created its workspace yet, so the choice would be stored under "${pendingForkOf}". Send a message first.`, true ) return } if (!setSkillEnabled(ws, path, enabled)) { - sendUserToast('Could not save the selection for this account.', true) + sendUserToast('Could not save this choice for this account.', true) return } const row = this.#row(path) if (row) row.enabled = enabled - // Whether people select skills at all. Never the skill itself: a path is + // Whether people turn skills off at all. Never the skill itself: a path is // workspace-authored text. logFeatureUsage('ai_session', 'skill_toggle', { key: enabled ? 'on' : 'off', workspace: ws }) - // The prompt lists exactly the enabled skills, so it has to be rebuilt + // The prompt lists exactly the skills that are on, so it has to be rebuilt // before the next message rather than on the next mode change. await this.#manager.refreshGlobalSkills(ws) } @@ -103,10 +103,10 @@ export class SkillsMenu { void this.#load(ws) } const ambiguous = ambiguousSkillNames(this.#rows) - // Enabled first: those are the ones a quick visit is most likely about. - const ordered = [...this.#rows].sort( - (a, b) => Number(b.enabled) - Number(a.enabled) || a.path.localeCompare(b.path) - ) + // By path. Ordering the ones that are on first would put every row in the same + // bucket now that skills start that way, and drop the one row it did move — a + // skill just turned off here — out of the shortcut that turns it back on. + const ordered = [...this.#rows].sort((a, b) => a.path.localeCompare(b.path)) const shown = ordered.slice(0, MAX_MENU_SKILLS) const manage = () => { closeMenu?.() diff --git a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts index 42773873c2..fc019ee983 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts @@ -69,6 +69,7 @@ export const httpTriggerRequestSchema = z.object({ "wrap_body": z.boolean().describe("If true, wraps the request body in a 'body' parameter").optional(), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), "raw_string": z.boolean().describe("If true, passes the request body as a raw string instead of parsing as JSON").optional(), + "allowed_origins": z.array(z.string()).describe("Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method.").nullable().optional(), "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ diff --git a/frontend/src/lib/components/custom_ui.ts b/frontend/src/lib/components/custom_ui.ts index 9771521bf4..2a3e6a2176 100644 --- a/frontend/src/lib/components/custom_ui.ts +++ b/frontend/src/lib/components/custom_ui.ts @@ -9,7 +9,6 @@ export type FlowBuilderWhitelabelCustomUi = { export?: boolean history?: boolean aiBuilder?: boolean - tutorials?: boolean diff?: boolean extraDeployOptions?: boolean editableSummary?: boolean diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index a89abf4374..5005cc931c 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -14,6 +14,7 @@ import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' import { buildTableEditorValues, + type TableEditorForeignKey, type TableEditorValues } from './apps/components/display/dbtable/tableEditor' import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable' @@ -250,6 +251,10 @@ export type IDbSchemaOps = { schema?: string colDefs: TableMetadata }) => Promise + onFetchForeignKeys: (params: { + table: string + schema?: string + }) => Promise } /** Thrown by a schema op when the user declines the out-of-order run warning. @@ -396,6 +401,48 @@ export function dbSchemaOpsWithPreviewScripts({ } } + /** Resolves to [] when the database has no foreign key introspection + * (BigQuery) or the query fails: callers treat foreign keys as optional. */ + async function fetchForeignKeys({ + table, + schema + }: { + table: string + schema?: string + }): Promise { + if (dbType === 'bigquery') return [] + try { + const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) + const fkResult = await runScriptAndPollResult({ + workspace, + requestBody: { args: dbArg, content: fkContent, language, tag } + }) + + let rawForeignKeys: RawForeignKey[] + if (dbType === 'snowflake') { + rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) + } else { + rawForeignKeys = fkResult as RawForeignKey[] + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + rawForeignKeys = rawForeignKeys.map((fk) => { + const lowerFk: any = {} + Object.keys(fk).forEach((key) => { + lowerFk[key.toLowerCase()] = fk[key] + }) + return lowerFk + }) + } + } + + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + return transformForeignKeys(rawForeignKeys) + } + } catch (e) { + console.warn('Failed to fetch foreign keys:', e) + } + return [] + } + return { onDelete: async ({ tableKey, schema }) => { const content = makeMarker('DROP_TABLE', { table: tableKey, schema }) @@ -454,44 +501,11 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { - let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] = - [] + const foreignKeys = await fetchForeignKeys({ table, schema }) let pk_constraint_name: string | undefined - // Fetch foreign keys (not supported for BigQuery) - if (dbType !== 'bigquery') { - try { - const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) - const fkResult = await runScriptAndPollResult({ - workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } - }) - - let rawForeignKeys: RawForeignKey[] - if (dbType === 'snowflake') { - rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) - } else { - rawForeignKeys = fkResult as RawForeignKey[] - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - rawForeignKeys = rawForeignKeys.map((fk) => { - const lowerFk: any = {} - Object.keys(fk).forEach((key) => { - lowerFk[key.toLowerCase()] = fk[key] - }) - return lowerFk - }) - } - } - - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - foreignKeys = transformForeignKeys(rawForeignKeys) - } - } catch (e) { - console.warn('Failed to fetch foreign keys:', e) - } - } - // Fetch primary key constraint name (not supported for BigQuery/MySQL) if (dbType !== 'bigquery' && dbType !== 'mysql') { try { diff --git a/frontend/src/lib/components/dbt/DbtEditor.svelte b/frontend/src/lib/components/dbt/DbtEditor.svelte index f12e510c61..4b30f8fa28 100644 --- a/frontend/src/lib/components/dbt/DbtEditor.svelte +++ b/frontend/src/lib/components/dbt/DbtEditor.svelte @@ -8,7 +8,7 @@ // single file, so the arguments, the run and the graph are all the project's // whichever file happens to be open. import { untrack } from 'svelte' - import { createEventDispatcher, onDestroy } from 'svelte' + import { createEventDispatcher, onDestroy, onMount } from 'svelte' import type { Schema, SupportedLanguage } from '$lib/common' import type { Preview, ScriptModule } from '$lib/gen' import { workspaceStore } from '$lib/stores' @@ -31,6 +31,15 @@ AssetGraphNodeData, DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import { + useDbtColumnLineage, + type DbtGraphPin + } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + EMPTY_COLUMN_GRAPH, + mergeColumnGraphs, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { DBT_DESCRIPTOR, DBT_MODULE_EXTENSIONS, @@ -97,6 +106,17 @@ } }) + // The stored schema can predate the parser (a CLI push, an older version), and + // the autosave baseline is taken as the mounted editor holds it + // (`schemaAsEditorMounts`), so the descriptor is re-inferred on mount rather + // than only on its first edit — otherwise the two only agree once edited. + onMount(async () => { + await inferSchema(code) + // Same single retry as ScriptEditor: the first parse can lose a transient + // wasm init race, and the baseline (`schemaAsEditorMounts`) retries too. + if (!validDescriptor && code) await inferSchema(code) + }) + function flushOpenFile() { if (openFile !== null && modules?.[openFile]) { modules[openFile] = { ...modules[openFile], content: editorCode } @@ -212,6 +232,28 @@ // the deployed graph, which previews by version instead. Either way the rows // come from the project whose SQL is displayed above them. let selectedBuffer = $state(undefined) + // Which graph the selection came from, so the lineage fetched below is the + // selected node's own project rather than whatever is deployed. + let selectionPin = $state(undefined) + // The selected model's column lineage, fetched on selection. Its own request + // rather than a field on the graph: only a project that opted into the + // analysis pass has any, and it is drawn for one model at a time. + const columnLineage = useDbtColumnLineage({ + workspace: () => opWs, + assetPaths: () => { + const path = selectedDbt ? selectedAsset?.path : undefined + return path ? [path] : [] + }, + pin: () => selectionPin + }) + // What the scripts around this project declare about its columns, off the + // same graph response the canvas drew. Merged rather than chosen between: a + // model's column and the ducklake column a script derives from it are one + // chain, and the trace has to cross that boundary. + let selectionProducerColumns = $state(EMPTY_COLUMN_GRAPH) + let selectionColumnGraph = $derived( + mergeColumnGraphs(columnLineage.graph, selectionProducerColumns) + ) let jobLoader: JobLoader | undefined = $state(undefined) let testJob: any = $state(undefined) @@ -514,10 +556,12 @@ testRunning={testIsLoading} testResult={testJob?.result} selection={graphSelection} - onSelect={(sel, dbt, buffer) => { + onSelect={(sel, dbt, buffer, pin, producerColumns) => { graphSelection = sel selectedDbt = dbt selectedBuffer = buffer + selectionPin = pin + selectionProducerColumns = producerColumns }} /> @@ -539,6 +583,10 @@ {args} fileInBundle={!!selectedDbt.original_file_path && !!modules?.[selectedDbt.original_file_path]} + columnGraph={selectionColumnGraph} + columnLoading={columnLineage.loading} + columnTruncated={columnLineage.truncated} + columnFailed={columnLineage.failed} onOpenFile={open} onClose={() => (graphSelection = undefined)} /> diff --git a/frontend/src/lib/components/dbt/DbtModelDetails.svelte b/frontend/src/lib/components/dbt/DbtModelDetails.svelte index 2d25dbdc00..5863d00a59 100644 --- a/frontend/src/lib/components/dbt/DbtModelDetails.svelte +++ b/frontend/src/lib/components/dbt/DbtModelDetails.svelte @@ -12,6 +12,9 @@ import { ClipboardCopy, Code2, FileCode2, Loader2, TableProperties, X } from 'lucide-svelte' import { copyToClipboard } from '$lib/utils' import type { DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import ColumnTraceSection from '$lib/components/assets/AssetGraph/ColumnTraceSection.svelte' + import DbtColumnList from '$lib/components/assets/AssetGraph/DbtColumnList.svelte' + import type { ColumnLineageGraph } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { previewDbtRows, type DbtPreview, type DbtPreviewBuffer } from './previewRows' import { nodeSelector } from './parseDbtRun' @@ -34,6 +37,13 @@ args, /** Whether this model's file is in the bundle being edited. */ fileInBundle = false, + /** The project's column-level lineage, when the descriptor asked for it. + * Fetched for this relation against the same graph the canvas draws, so + * the trace and the nodes above it describe one parse. */ + columnGraph, + columnLoading = false, + columnTruncated = false, + columnFailed = false, onOpenFile, onClose }: { @@ -45,6 +55,10 @@ buffer?: DbtPreviewBuffer args?: Record fileInBundle?: boolean + columnGraph?: ColumnLineageGraph + columnLoading?: boolean + columnTruncated?: boolean + columnFailed?: boolean onOpenFile?: (path: string) => void onClose?: () => void } = $props() @@ -111,7 +125,9 @@ return typeof v === 'object' ? JSON.stringify(v) : String(v) } - let columns = $derived(Object.entries(dbt.columns ?? {})) + let hasColumns = $derived( + !!dbt.column_schema?.length || Object.keys(dbt.columns ?? {}).length > 0 + ) // `dbt show` SELECTs from the node's own relation and the worker intersects // the selector with `resource_type:model`, so offering it on a seed, snapshot // or source only ever produces a failed job. @@ -202,15 +218,15 @@ {#if stalePlaceholders}
- The run arguments have changed since this graph was parsed, so these rows need not - describe the models on screen — arguments reach schemas, aliases and which models exist - at all. Refresh the models to draw and preview them under the current ones. + The run arguments have changed since this graph was parsed, so these rows need not describe + the models on screen — arguments reach schemas, aliases and which models exist at all. Refresh + the models to draw and preview them under the current ones.
{:else if staleVars}
- The run form's vars have changed since this graph was parsed. Rows are previewed under - the vars it was parsed with, so they still describe the models on screen — refresh the - models to draw and preview them under the current ones. + The run form's vars have changed since this graph was parsed. Rows are previewed under the + vars it was parsed with, so they still describe the models on screen — refresh the models to + draw and preview them under the current ones.
{/if} @@ -234,26 +250,9 @@ {/if} - {#if columns.length > 0 || (dbt.data_tests?.length ?? 0) > 0} + {#if hasColumns || (dbt.data_tests?.length ?? 0) > 0}
- {#if columns.length > 0} -
-
columns declared
-
- {#each columns as [name, desc] (name)} -
- {name} - {desc} -
- {/each} -
- -
- Declared metadata — dbt reports no column-level lineage. -
-
- {/if} + {#if (dbt.data_tests?.length ?? 0) > 0}
tests
@@ -269,6 +268,16 @@
{/if} + + {#if showRows && preview} {#if 'error' in preview}
{preview.error}
diff --git a/frontend/src/lib/components/dbt/DbtModelGraph.svelte b/frontend/src/lib/components/dbt/DbtModelGraph.svelte index e6d948d926..bd7f42a9e7 100644 --- a/frontend/src/lib/components/dbt/DbtModelGraph.svelte +++ b/frontend/src/lib/components/dbt/DbtModelGraph.svelte @@ -25,6 +25,12 @@ DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' import { useDbtRunStatus } from './runStatus.svelte' + import type { DbtGraphPin } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + buildColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' let { workspace, @@ -80,7 +86,19 @@ * buffer rather than a deployed version — as submitted, not as the * editor holds it now. Sent with the selection rather than exposed on * its own so it can never disagree with the SQL the parent shows. */ - buffer: DbtPreviewBuffer | undefined + buffer: DbtPreviewBuffer | undefined, + /** Which graph this node was taken from, so anything else fetched + * about it describes the same project: the editor's own parse job + * when the panel is pinned to one, else the deployed version. Sent + * with the selection for the same reason the buffer is — it must not + * be able to disagree with the node on screen. */ + pin: DbtGraphPin, + /** Column lineage the CONSUMERS of this project declare — a script + * reading a model's column and writing a ducklake one. It comes off + * the same graph response, and the details pane merges it with the + * project's own so a trace crosses that boundary instead of ending + * at it. */ + producerColumns: ColumnLineageGraph ) => void } = $props() @@ -364,6 +382,25 @@ // graph that actually came back. let editorParsed = $derived(refreshJob != undefined && raw?.dbt_snapshot_job === refreshJob) + // Which stored graph is on screen. Anything the details pane fetches about a + // selected node asks for this one, so it cannot describe a node parsed from + // the buffer with the deployed version's answer. + let pin = $derived( + editorParsed && refreshJob ? { jobId: refreshJob } : { scriptHash: deployedHash } + ) + + // What the scripts around this project declare about its columns. Empty for + // the ordinary project nothing downstream annotates. + // + // A consumer is anchored here only by its `// materialize` target: this graph + // is fetched `asset_kinds=dbt` so the canvas is the project and nothing else, + // and `buildColumnGraph`'s other anchor is a ducklake WRITE EDGE, which that + // filter drops. So a script consuming a model and writing a ducklake table it + // never declared contributes no hop in this editor, while it does on the + // pipeline page, whose graph spans both kinds. Widening the request would put + // ducklake nodes on the dbt canvas, which is the opposite of what it is for. + let producerColumns = $derived(graph ? buildColumnGraph(graph) : EMPTY_COLUMN_GRAPH) + // `untrack`, because the effect that reloads the graph clears the selection // through here: reading the graph to describe a selection would subscribe that // effect to the very state its own fetch writes, and it would reload forever. @@ -374,7 +411,9 @@ sel?.kind === 'asset' ? graph?.assets.find((a) => a.kind === sel.asset_kind && a.path === sel.path)?.dbt : undefined, - editorParsed ? parsedBuffer : undefined + editorParsed ? parsedBuffer : undefined, + pin, + producerColumns ) ) } @@ -405,7 +444,6 @@ if (deployedHash != undefined) return 'as of last deploy' return 'never parsed' }) -
@@ -435,8 +473,8 @@ {#if refreshPending}
- Still parsing. A cold worker provisions the dbt engine before it starts; a project - pinned to a worker tag nothing serves waits here indefinitely. + Still parsing. A cold worker provisions the dbt engine before it starts; a project pinned to a + worker tag nothing serves waits here indefinitely. { - const modules = flow?.value?.modules - const lastModule = modules && modules.length > 0 ? modules[modules.length - 1] : undefined - if (lastModule?.value?.type !== 'aiagent') return false - return agentStreamingEnabled(lastModule.value) - }) @@ -749,7 +742,6 @@ onRunFlow={runFlowForChat} {deploymentInProgress} path={flow?.path ?? ''} - useStreaming={shouldUseStreaming} inputSchema={flow?.schema} /> {:else} diff --git a/frontend/src/lib/components/details/ScriptDetail.svelte b/frontend/src/lib/components/details/ScriptDetail.svelte index c20e0b3d09..835bb6fdd0 100644 --- a/frontend/src/lib/components/details/ScriptDetail.svelte +++ b/frontend/src/lib/components/details/ScriptDetail.svelte @@ -22,7 +22,13 @@ } from '$lib/utils' import Tooltip from '$lib/components/Tooltip.svelte' import ShareModal from '$lib/components/ShareModal.svelte' - import { enterpriseLicense, userStore, userWorkspaces } from '$lib/stores' + import { + disableHubStore, + enterpriseLicense, + hubBaseUrlStore, + userStore, + userWorkspaces + } from '$lib/stores' import { isDeployable, ALL_DEPLOYABLE } from '$lib/utils_deployable' import AIFormAssistant from '$lib/components/copilot/AIFormAssistant.svelte' @@ -59,6 +65,7 @@ Eye, FolderOpen, GitFork, + Globe2, History, Loader2, Pen, @@ -71,6 +78,7 @@ ChevronDown, ChevronRight } from 'lucide-svelte' + import { scriptToHubUrl } from '$lib/hub' import SharedBadge from '$lib/components/SharedBadge.svelte' import Popover from '$lib/components/Popover.svelte' import ScriptVersionHistory from '$lib/components/ScriptVersionHistory.svelte' @@ -647,6 +655,17 @@ } }) + if (!$disableHubStore) { + menuItems.push({ + label: 'Publish to Hub', + Icon: Globe2, + onclick: () => { + if (!script) return + window.open(scriptToHubUrl(script, $hubBaseUrlStore).toString(), '_blank', 'noopener') + } + }) + } + if (isDeployable('script', script?.path ?? '', deployUiSettings)) { menuItems.push({ label: 'Deploy to staging/prod', diff --git a/frontend/src/lib/components/displayNameLoaders.ts b/frontend/src/lib/components/displayNameLoaders.ts new file mode 100644 index 0000000000..5eed1e81a0 --- /dev/null +++ b/frontend/src/lib/components/displayNameLoaders.ts @@ -0,0 +1,71 @@ +import { get } from 'svelte/store' +import { IntegrationService, ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { addResourceTypeDisplayName, setHubIntegrationDisplayNames } from './resourceTypeDisplay' + +/** + * Loads what `resourceTypeDisplayName` and `integrationDisplayName` read: a type's stored name for a + * surface that holds no row for it, and the hub's integration list, which every picker that needs + * it shares. Apart from `resourceTypeDisplay`, which makes no API calls so it can be unit-tested + * alone. Cached briefly: drawers and pickers reopen often, and a name rarely changes. + */ +const CACHE_MS = 60_000 + +const resourceTypeRowCached = createCache( + ({ workspace, name }: { workspace: string; name: string }) => + ResourceService.getResourceType({ workspace, path: name }).then( + (rt) => addResourceTypeDisplayName(rt), + () => {} + ), + { invalidateMs: CACHE_MS, maxSize: 50 } +) + +/** + * Fill `resourceTypeDisplayName` for one type, for a surface titled with a type it holds no row + * for. The name is stored with the type, so this reads the row rather than the hub. + */ +export function loadResourceTypeDisplayName(workspace: string, name: string): Promise { + return resourceTypeRowCached({ workspace, name }) +} + +/** Bumped by every failed read, so the next caller keys a fresh read rather than the rejection. */ +let failedReads = 0 + +const hubIntegrationsCached = createCache( + ({ kind }: { kind?: string; refresh: number; attempt: number }) => + IntegrationService.listHubIntegrations({ kind }).then( + (integrations) => { + setHubIntegrationDisplayNames(integrations) + return integrations + }, + (error) => { + failedReads += 1 + throw error + } + ), + { invalidateMs: CACHE_MS } +) + +/** + * The hub's integration list, read once a minute per `kind` however many pickers ask, recording + * each integration's name on the way. A failed read rejects, so a picker can say the hub is + * unavailable, but is not kept: the next caller reads again. `refresh` is a picker's refresh + * count, and a new value reads again too. + */ +export function listHubIntegrationsShared(kind?: string, refresh = 0) { + return hubIntegrationsCached({ kind, refresh, attempt: failedReads }) +} + +/** + * Fill `integrationDisplayName` for a picker whose integrations come from its own items rather + * than the hub's integration list, as the hub app and flow pickers do. Unfiltered: `kind` + * narrows by script kind, so asking for an app or a flow would name nothing. + */ +export function loadHubIntegrationDisplayNames(): Promise { + if (get(disableHubStore)) return Promise.resolve() + return listHubIntegrationsShared().then( + () => {}, + () => {} + ) +} diff --git a/frontend/src/lib/components/editorStoragePathContext.ts b/frontend/src/lib/components/editorStoragePathContext.ts new file mode 100644 index 0000000000..e3685995c4 --- /dev/null +++ b/frontend/src/lib/components/editorStoragePathContext.ts @@ -0,0 +1,19 @@ +import { getContext, setContext } from 'svelte' + +// The path an editor's draft is stored under, the same key the live-editor draft +// registers. Published by whichever ancestor owns it (the session tab, or the +// full-page editor) because the parts that must say which item an editor is open +// on sit below both, and the path they can see themselves is the renamed one. + +const KEY = 'EditorStoragePath' + +/** `undefined` where the editor has no stored draft to speak of (a drawer mount). */ +export type EditorStoragePath = () => string | undefined + +export function setEditorStoragePath(storagePath: EditorStoragePath): void { + setContext(KEY, storagePath) +} + +export function getEditorStoragePath(): EditorStoragePath | undefined { + return getContext(KEY) +} diff --git a/frontend/src/lib/components/flows/FlowEditor.svelte b/frontend/src/lib/components/flows/FlowEditor.svelte index 812e971da9..bf25a1dcda 100644 --- a/frontend/src/lib/components/flows/FlowEditor.svelte +++ b/frontend/src/lib/components/flows/FlowEditor.svelte @@ -4,6 +4,7 @@ import FlowEditorPanel from './content/FlowEditorPanel.svelte' import { agentEditorTarget, type AgentEditorTarget } from './agentEditorStore.svelte' import AgentEditorModal from './content/AgentEditorModal.svelte' + import { repointLinkedAgent } from './linkedAgentDrafts' import FlowModuleSchemaMap from './map/FlowModuleSchemaMap.svelte' import type { OpenInSessionSource } from '$lib/components/sessions/OpenInSessionButton.svelte' import WindmillIcon from '../icons/WindmillIcon.svelte' @@ -55,7 +56,6 @@ interface Props { loading: boolean disableStaticInputs?: boolean - disableTutorials?: boolean disableAi?: boolean disableSettings?: boolean disabledFlowInputs?: boolean @@ -99,7 +99,6 @@ let { loading, disableStaticInputs = false, - disableTutorials = false, disableAi = false, disableSettings = false, disabledFlowInputs = false, @@ -391,7 +390,6 @@ bind:this={flowModuleSchemaMap} controlsPosition={compactGraphOverlay ? 'bottom' : 'top'} {disableStaticInputs} - {disableTutorials} {disableAi} {disableSettings} {smallErrorHandler} @@ -554,4 +552,5 @@ t.host?.flowPath === $pathStore && targetWorkspace(t) === editorWorkspace} + onRenamed={(from, to) => repointLinkedAgent(flowStore.val.value, from, to)} /> diff --git a/frontend/src/lib/components/flows/FlowEditorTutorial.svelte b/frontend/src/lib/components/flows/FlowEditorTutorial.svelte deleted file mode 100644 index ca1972914e..0000000000 --- a/frontend/src/lib/components/flows/FlowEditorTutorial.svelte +++ /dev/null @@ -1,62 +0,0 @@ - - -{#key $tutorialsToDo} - - {#snippet buttonReplacement()} -
{#if showDetail && (brainParams.length > 0 || inheritedTools.length > 0)} @@ -506,7 +619,32 @@ {/if}
- {#if !providerOk} + {#if unavailable === 'forbidden'} +
+ + You don't have access to {agent}, so its configuration + can't be shown or edited here. + +
+ {:else if unavailable === 'missing'} +
+ + No saved agent exists at {agent}. It may have been + renamed or deleted. Remove the link to configure the step here, or add the agent again + from Saved agents. +
+ +
+ +
+ {:else if !providerOk}
This agent's model provider{#if providerPath} diff --git a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte index feffb327c4..2e7b6469f7 100644 --- a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte +++ b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte @@ -15,10 +15,20 @@ openFieldsByStep.delete(oldest) } } + + /** + * The rows this step's form has open, for the run form, which has no add-field control of its + * own and would otherwise not offer a field that was added here and left at its default: to a + * reader of the stored transforms alone, that is indistinguishable from a field nobody touched. + */ + export function openAgentFields(key: string | undefined): string[] { + return (key ? openFieldsByStep.get(key) : undefined) ?? [] + }
@@ -134,6 +145,34 @@ {/if} + - import { workspaceStore } from '$lib/stores' - import { createFlowChatManager } from './FlowChatManager.svelte' + import { enterpriseLicense, workspaceStore } from '$lib/stores' + import { sendUserToast } from '$lib/toast' + import { createChat, type Chat, type ChatState } from 'windmill-chat' import FlowConversationsSidebar from './FlowConversationsSidebar.svelte' import FlowChatInterface from './FlowChatInterface.svelte' - import { getContext, untrack } from 'svelte' + import { getContext } from 'svelte' import type { FlowEditorContext } from '../types' interface Props { + /** + * Runs the flow for one turn and returns the job id: the deployed flow on the + * flow page, a preview run in the editor. The run must carry `memory_id` = + * `conversationId`, which is what ties the job to the conversation. + */ onRunFlow: ( userMessage: string, conversationId: string, additionalInputs?: Record ) => Promise - useStreaming?: boolean deploymentInProgress?: boolean path: string hideSidebar?: boolean @@ -22,34 +27,46 @@ let { onRunFlow, deploymentInProgress = false, - useStreaming = false, path, hideSidebar = false, inputSchema = undefined }: Props = $props() const flowEditorContext = getContext('FlowEditorContext') + // The editor may act on a workspace other than the nav store's (AI-session live editor). + const workspace = $derived(flowEditorContext?.opWorkspace?.() ?? $workspaceStore) - const manager = createFlowChatManager() - manager.operatingWorkspace = () => flowEditorContext?.opWorkspace?.() + let chat = $state(undefined) + let chatState = $state(undefined) + let sidebar = $state(undefined) - // Initialize manager when component mounts $effect(() => { - if ($workspaceStore) { - manager.initialize(onRunFlow, path, useStreaming) - } - + const ws = workspace + const flowPath = path + if (!ws || !flowPath) return + const created = createChat({ + flowPath, + workspace: ws, + baseUrl: window.location.origin, + history: 'server', + // Only an enterprise server honours it; elsewhere it would just log a warning per + // poll. The license loads asynchronously, so a cold load may create the chat twice. + pollDelayMs: $enterpriseLicense ? 50 : undefined, + run: async ({ user_message, ...inputs }, { conversationId }) => { + const jobId = await onRunFlow(String(user_message), conversationId, inputs) + if (!jobId) throw new Error('the flow did not start') + // The server creates the conversation with the run, so the sidebar can list + // it now, whatever becomes of the turn. + sidebar?.conversationStarted(conversationId) + return jobId + }, + onError: (error) => sendUserToast('Failed to run flow: ' + error.message, true) + }) + const unsubscribe = created.subscribe((s) => (chatState = s)) + chat = created return () => { - manager.cleanup() - } - }) - - // Initialize InfiniteList when component mounts or flowPath changes - $effect(() => { - if ($workspaceStore && path && manager.conversationListComponent) { - untrack(() => { - manager.setupInfiniteList() - }) + unsubscribe() + created.destroy() } }) @@ -69,8 +86,17 @@
- {#if !hideSidebar} - + {#if chat && chatState} + {#if !hideSidebar} + + {/if} + {/if} -
diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index ab52d06102..22fe037728 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -3,19 +3,76 @@ import { MessageCircle, Loader2, Settings2 } from 'lucide-svelte' import ChatMessage from '$lib/components/chat/ChatMessage.svelte' import ChatInput from '$lib/components/chat/ChatInput.svelte' - import { FlowChatManager } from './FlowChatManager.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import SchemaForm from '$lib/components/SchemaForm.svelte' import { type DynamicInput } from '$lib/utils' + import { tick, untrack } from 'svelte' + import type { Chat, ChatState } from 'windmill-chat' interface Props { - manager: FlowChatManager + chat: Chat + chatState: ChatState deploymentInProgress?: boolean additionalInputsSchema?: Record path: string + workspace?: string } - let { manager, deploymentInProgress = false, additionalInputsSchema, path }: Props = $props() + let { + chat, + chatState, + deploymentInProgress = false, + additionalInputsSchema, + path, + workspace = undefined + }: Props = $props() + + let inputMessage = $state('') + let inputElement = $state(undefined) + let messagesContainer = $state(undefined) + let loadingOlder = false + + const busy = $derived(chatState.status === 'submitted' || chatState.status === 'streaming') + // Deriveds notify only when their value changes; `chatState` itself is a new + // object on every token, and following it would drag a reader who scrolled up + // back to the end on each one. + const messageCount = $derived(chatState.messages.length) + const conversationId = $derived(chatState.conversationId) + const loadingMessages = $derived(chatState.loadingMessages) + + // Follow the conversation: new messages and a conversation switch scroll to the + // end, older pages loaded at the top keep the viewport where it was. + $effect(() => { + messageCount + conversationId + loadingMessages + untrack(() => { + if (loadingOlder) return + tick().then(() => { + if (messagesContainer) messagesContainer.scrollTop = messagesContainer.scrollHeight + }) + }) + }) + + async function handleScroll() { + if ( + !messagesContainer || + !chatState.hasMoreMessages || + chatState.loadingMessages || + loadingOlder + ) + return + if (messagesContainer.scrollTop > 10) return + loadingOlder = true + const previousHeight = messagesContainer.scrollHeight + try { + await chat.loadOlderMessages() + await tick() + messagesContainer.scrollTop = messagesContainer.scrollHeight - previousHeight + } finally { + loadingOlder = false + } + } // Derive helperScript for dynamic inputs from schema const dynamicInputHelperScript = $derived.by((): DynamicInput.HelperScript | undefined => { @@ -63,11 +120,17 @@ showInputsModal = false } - function handleSendMessage() { + async function handleSendMessage() { + const text = inputMessage.trim() + if (!text || busy || deploymentInProgress) return const inputs = additionalInputsSchema ? (loadInputsFromStorage() ?? additionalInputsValues) : undefined - manager.sendMessage(inputs) + inputMessage = '' + // A failure is reported through the chat's `onError` and as a failed message. + await chat.sendMessage(text, { inputs }).catch(() => {}) + await tick() + inputElement?.focus() } function openInputsModal() { @@ -93,7 +156,7 @@ schema={additionalInputsSchema} bind:args={additionalInputsValues} helperScript={dynamicInputHelperScript} - workspace={manager.operatingWorkspace?.()} + {workspace} /> {#snippet actions()} @@ -104,18 +167,18 @@
{#if deploymentInProgress} {/if} - {#if manager.isLoadingMessages} + {#if chatState.loadingMessages && chatState.messages.length === 0}
- {:else if manager.messages.length === 0} + {:else if chatState.messages.length === 0}

Start a conversation

@@ -123,16 +186,15 @@
{:else}
- {#each manager.messages as message (message.id)} + {#each chatState.messages as message (message.id)} {/each} - {#if manager.isWaitingForResponse} + {#if busy}
Processing... @@ -148,7 +210,7 @@
diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts deleted file mode 100644 index 7cf2c8d6f7..0000000000 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ /dev/null @@ -1,684 +0,0 @@ -import type { FlowConversation, FlowConversationMessage } from '$lib/gen/types.gen' -import { FlowConversationsService, JobService } from '$lib/gen' -import { sendUserToast } from '$lib/toast' -import { waitJob } from '$lib/components/waitJob' -import { tick } from 'svelte' -import InfiniteList from '$lib/components/InfiniteList.svelte' -import { workspaceStore, userStore } from '$lib/stores' -import { get } from 'svelte/store' -import { parseStreamDeltas } from '$lib/components/chat/utils' -import { randomUUID } from '$lib/utils/uuid' - -export interface ChatMessage extends FlowConversationMessage { - loading?: boolean - streaming?: boolean -} - -export interface ConversationWithDraft extends FlowConversation { - isDraft?: boolean -} - -export class FlowChatManager { - // State - messages = $state([]) - inputMessage = $state('') - isLoading = $state(false) - isLoadingMessages = $state(false) - isWaitingForResponse = $state(false) - messagesContainer = $state(undefined) - inputElement = $state(undefined) - page = $state(1) - hasMoreMessages = $state(false) - loadingMoreMessages = $state(false) - currentEventSource = $state(undefined) - pollingInterval = $state | undefined>(undefined) - currentJobId = $state(undefined) - conversations = $state([]) - deletingConversationId = $state(undefined) - isSidebarExpanded = $state(false) - selectedConversationId = $state(undefined) - conversationListComponent = $state(undefined) - - // Private state - #conversationsCache = $state>({}) - #scrollTimeout: ReturnType | undefined = undefined - #perPage = 50 - - // Options - #onRunFlow?: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise - #useStreaming = $state(false) - #path = $state(undefined) - - // When the flow editor runs as an AI-session live editor, it acts on a workspace - // that can differ from the nav store. FlowChat.svelte wires this to - // FlowEditorContext.opWorkspace so workspace-scoped calls hit the acting workspace. - operatingWorkspace?: () => string | undefined - - #workspace(): string | undefined { - return this.operatingWorkspace?.() ?? get(workspaceStore) - } - - initialize( - onRunFlow: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise, - path: string, - useStreaming: boolean = false - ) { - this.#onRunFlow = onRunFlow - this.#path = path - this.#useStreaming = useStreaming - } - - updateConversationId(conversationId: string | undefined) { - this.selectedConversationId = conversationId - } - - cleanup() { - if (this.currentEventSource) { - this.currentEventSource.close() - this.currentEventSource = undefined - } - this.stopPolling() - this.isLoading = false - this.isWaitingForResponse = false - this.currentJobId = undefined - } - - // Public methods for component to call - fillInputMessage(message: string) { - this.inputMessage = message - } - - focusInput() { - this.inputElement?.focus() - } - - clearMessages() { - this.messages = [] - this.inputMessage = '' - this.page = 1 - } - - async createConversation({ clearMessages = true }: { clearMessages?: boolean }) { - // Check if there's already a draft conversation - const existingDraft = this.conversations.find((c) => c.isDraft) - if (existingDraft) { - // Select the existing draft instead of creating a new one - this.selectedConversationId = existingDraft.id - this.clearMessages() - return existingDraft.id - } - const newConversationId = randomUUID() - this.selectedConversationId = newConversationId - - // Create a new conversation object and add it to the top of the list - const newConversation: ConversationWithDraft = { - id: newConversationId, - workspace_id: this.#workspace()!, - flow_path: this.#path!, - title: 'New chat', - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - created_by: get(userStore)!.username!, - isDraft: true - } - - // Prepend to conversations list - this.conversations = [newConversation, ...this.conversations] - // Clear messages in the chat interface - if (clearMessages) { - this.clearMessages() - } - this.focusInput() - - return newConversationId - } - - setupInfiniteList() { - this.conversationListComponent?.setLoader((page, perPage) => - this.loadConversations(page, perPage) - ) - this.conversationListComponent?.setDeleteItemFn((id) => this.deleteConversation(id)) - } - - async selectConversation(conversationId: string, isDraft?: boolean) { - this.selectedConversationId = conversationId - // Load conversation messages into chat interface - if (isDraft) { - // For draft conversations, just clear messages (don't try to load from backend) - this.clearMessages() - } else { - // For persisted conversations, load messages from backend - await this.loadConversationMessages(conversationId) - } - } - - async refreshConversations() { - await this.conversationListComponent?.loadData('forceRefresh') - } - - // Only used by InfiniteList - private async deleteConversation(conversationId: string) { - try { - this.deletingConversationId = conversationId - await FlowConversationsService.deleteFlowConversation({ - workspace: this.#workspace()!, - conversationId - }) - if (this.selectedConversationId === conversationId) { - this.selectedConversationId = undefined - this.clearMessages() - } - sendUserToast('Conversation deleted successfully') - } catch (error) { - console.error('Failed to delete conversation:', error) - sendUserToast('Failed to delete conversation', true) - throw error - } finally { - this.deletingConversationId = undefined - } - } - - async cancelCurrentJob() { - if (!this.#workspace()) { - return - } - - try { - if (this.currentJobId) { - await JobService.cancelQueuedJob({ - workspace: this.#workspace()!, - id: this.currentJobId, - requestBody: {} - }) - sendUserToast(`Job ${this.currentJobId} cancelled`) - } - } catch (error) { - console.error('Error cancelling job:', error) - sendUserToast('Could not cancel job', true) - } finally { - this.cleanup() - } - } - - async loadConversationMessages(conversationId?: string) { - this.page = 1 - await this.loadMessages(true, conversationId) - } - - // Only used by InfiniteList - private async loadConversations(page: number, perPage: number) { - if (!this.#workspace() || !this.#path) return [] - - try { - const response = await FlowConversationsService.listFlowConversations({ - workspace: this.#workspace()!, - flowPath: this.#path, - page: page, - perPage: perPage - }) - - return response - } catch (error) { - console.error('Failed to load conversations:', error) - sendUserToast('Failed to load conversations', true) - return [] - } - } - - // Message loading - private async loadMessages(reset: boolean, conversationId?: string) { - let conversationIdToUse = conversationId ?? this.selectedConversationId - if (!this.#workspace() || !conversationIdToUse) return - - if (reset) { - if (this.#conversationsCache[conversationIdToUse]) { - this.messages = this.#conversationsCache[conversationIdToUse] - return - } - this.isLoadingMessages = true - } else { - this.loadingMoreMessages = true - } - - const pageToFetch = reset ? 1 : this.page + 1 - - try { - const previousScrollHeight = this.messagesContainer?.scrollHeight || 0 - - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationIdToUse, - page: pageToFetch, - perPage: this.#perPage - }) - - if (reset) { - this.#conversationsCache[conversationIdToUse] = response - this.messages = response - this.isLoadingMessages = false - await new Promise((resolve) => setTimeout(resolve, 100)) - this.scrollToBottom() - } else { - this.messages = [...response, ...this.messages] - this.page = pageToFetch - // Restore scroll position - await new Promise((resolve) => setTimeout(resolve, 50)) - if (this.messagesContainer) { - this.messagesContainer.scrollTop = - this.messagesContainer.scrollHeight - previousScrollHeight - } - } - - this.hasMoreMessages = response.length === this.#perPage - } catch (error) { - console.error('Failed to load messages:', error) - sendUserToast('Failed to load messages: ' + error) - } finally { - this.isLoadingMessages = false - this.loadingMoreMessages = false - } - } - - handleScroll = () => { - if (this.#scrollTimeout) clearTimeout(this.#scrollTimeout) - - this.#scrollTimeout = setTimeout(() => { - if (!this.messagesContainer || !this.hasMoreMessages || this.loadingMoreMessages) return - - if (this.messagesContainer.scrollTop <= 10) { - this.loadMessages(false) - } - }, 200) - } - - scrollToBottom() { - if (this.messagesContainer) { - this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight - } - } - - private scrollToUserMessage(messageId: string) { - if (!this.messagesContainer) return - const messageElement = this.messagesContainer.querySelector(`[data-message-id="${messageId}"]`) - if (messageElement) { - messageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }) - } - } - - private getLastPersistedMessageSeq() { - for (let i = this.messages.length - 1; i >= 0; i--) { - const message = this.messages[i] - if (!message.id.startsWith('temp-')) { - return message.created_seq - } - } - - return undefined - } - - // Polling - private async pollJobResult(jobId: string) { - try { - await waitJob(jobId, this.#workspace()) - } catch (error) { - console.error('Error polling job result:', error) - } finally { - // Do a final poll to get all messages from database - try { - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - } catch {} - this.cleanup() - } - } - - private async pollConversationMessages( - conversationId: string, - options?: { isNewConversation?: boolean; removeTempMessages?: boolean } - ) { - if (!this.#workspace()) return - - try { - const lastSeq = this.getLastPersistedMessageSeq() - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationId, - page: 1, - perPage: 50, - afterSeq: lastSeq - }) - - if (options?.isNewConversation) { - await this.refreshConversations() - } - - const filteredResponse = response.filter((msg) => msg.message_type !== 'user') - for (const msg of filteredResponse) { - if (!this.messages.find((m) => m.id === msg.id)) { - this.messages = [...this.messages, msg] - } - } - - // Only remove temporary messages when explicitly requested (e.g., after job completion) - // During streaming, we keep temp messages to avoid them disappearing due to race conditions - if (options?.removeTempMessages) { - this.messages = this.messages.filter( - (msg) => !msg.id.startsWith('temp-') || msg.message_type === 'user' - ) - } - } catch (error) { - console.error('Polling error:', error) - } - } - - private startPolling(conversationId: string, isNewConversation?: boolean) { - if (this.pollingInterval) return - this.pollingInterval = setInterval(() => { - this.pollConversationMessages(conversationId, { isNewConversation }) - }, 500) // Poll every 0.5 seconds - setTimeout( - () => { - this.stopPolling() - }, - 2 * 60 * 1000 - ) // Stop polling after 2 minutes - } - - private stopPolling() { - if (this.pollingInterval) { - clearInterval(this.pollingInterval) - this.pollingInterval = undefined - } - } - - // Message sending - async sendMessage(additionalInputs?: Record) { - if (!this.inputMessage.trim() || this.isLoading) return - - const isNewConversation = this.messages.length === 0 - - // Reset state for new message - this.stopPolling() - - // Generate a new conversation ID if we don't have one - let currentConversationId = this.selectedConversationId - if (!this.selectedConversationId) { - const newConversationId = await this.createConversation({ clearMessages: false }) - currentConversationId = newConversationId - } - - if (!currentConversationId) { - console.error('No conversation ID found') - return - } - - // Invalidate the conversation cache - delete this.#conversationsCache[currentConversationId] - - const userMessage: ChatMessage = { - id: `temp-${randomUUID()}`, - content: this.inputMessage.trim(), - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'user', - conversation_id: currentConversationId - } - - this.messages = [...this.messages, userMessage] - const messageContent = this.inputMessage.trim() - this.inputMessage = '' - this.isLoading = true - this.isWaitingForResponse = true - - try { - await tick() - this.scrollToUserMessage(userMessage.id) - - if (this.#useStreaming && this.#path) { - await this.handleStreamingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } else { - await this.handlePollingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } - } catch (error) { - console.error('Error running flow:', error) - sendUserToast('Failed to run flow: ' + error, true) - } finally { - if (!this.#useStreaming) { - this.isLoading = false - } - } - - await tick() - this.focusInput() - } - - private async handleStreamingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - // Close any existing EventSource - if (this.currentEventSource) { - this.currentEventSource.close() - } - - // Track stream state for this message - let accumulatedContent = '' - let assistantMessageId = '' - let isCompleted = false - - try { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - - // Build the EventSource URL - const streamUrl = `/api/w/${this.#workspace()}/jobs_u/getupdate_sse/${jobId}` - const url = new URL(streamUrl, window.location.origin) - url.searchParams.set('poll_delay_ms', '50') - url.searchParams.set('fast', 'true') - url.searchParams.set('only_result', 'true') - // Create EventSource connection - const eventSource = new EventSource(url.toString()) - this.currentEventSource = eventSource - - // start polling - this.startPolling(currentConversationId, isNewConversation) - - eventSource.onmessage = async (event) => { - try { - const data = JSON.parse(event.data) - const type = data.type - - // Handle timeout - reconnect to SSE - if (type === 'timeout') { - eventSource.close() - this.currentEventSource = undefined - // Reconnect - this.handleStreamingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - return - } - - // Handle ping - just ignore - if (type === 'ping') { - return - } - - // Handle error - if (type === 'error') { - eventSource.close() - this.currentEventSource = undefined - console.error('SSE error:', data) - sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) - this.cleanup() - return - } - - // Handle not found - if (type === 'not_found') { - eventSource.close() - this.currentEventSource = undefined - console.error('Job not found') - sendUserToast('Job not found', true) - this.cleanup() - return - } - - if (type === 'update') { - if (data.flow_stream_job_id) { - this.currentJobId = data.flow_stream_job_id - } - // Process new stream content - if (data.new_result_stream) { - // Stop polling since we are receiving last step streaming - this.stopPolling() - const { - type, - content: newContent, - success - } = parseStreamDeltas(data.new_result_stream) - accumulatedContent += newContent - - // Create tool message if type is tool_result - if (type === 'tool_result') { - // set last message streaming to false - this.messages = this.messages.map((msg) => - msg.id === this.messages[this.messages.length - 1].id - ? { ...msg, streaming: false } - : msg - ) - - this.messages = [ - ...this.messages, - { - id: 'temp-' + randomUUID(), - content: newContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'tool', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: false, - success - } - ] - // Reset assistant message ID since we are creating a tool message - assistantMessageId = '' - accumulatedContent = '' - } - - // Create message on first content - else if ( - type === 'message' && - assistantMessageId.length === 0 && - accumulatedContent.length > 0 - ) { - assistantMessageId = 'temp-' + randomUUID() - this.messages = [ - ...this.messages, - { - id: assistantMessageId, - content: accumulatedContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'assistant', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: true - } - ] - } else { - // Update existing message - this.messages = this.messages.map((msg) => - msg.id === assistantMessageId ? { ...msg, content: accumulatedContent } : msg - ) - } - } - - // Handle completion - if (data.completed) { - isCompleted = true - // Do a final poll to get all messages from database - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - this.cleanup() - } - } - } catch (error) { - console.error('Error processing stream event:', error) - } - } - - eventSource.onerror = (error) => { - if (isCompleted) return - console.error('EventSource error:', error) - sendUserToast('Stream error occurred', true) - this.cleanup() - } - } catch (error) { - console.error('Stream connection error:', error) - sendUserToast('Failed to connect to stream', true) - this.cleanup() - } - } - - private async handlePollingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - - // Store the current job ID so it can be cancelled - this.currentJobId = jobId - - if (isNewConversation) { - await this.refreshConversations() - } - - // Start polling for intermediate messages in non-streaming mode too - this.startPolling(currentConversationId) - this.pollJobResult(jobId) - } -} - -export const createFlowChatManager = () => new FlowChatManager() diff --git a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte index 2f09bcebe1..02c2581db1 100644 --- a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte @@ -1,26 +1,72 @@
@@ -31,11 +77,11 @@ unifiedSize="md" variant="subtle" startIcon={{ - icon: manager.isSidebarExpanded ? PanelLeftClose : PanelLeftOpen, + icon: expanded ? PanelLeftClose : PanelLeftOpen, classes: 'ml-[2px]' }} - onClick={() => (manager.isSidebarExpanded = !manager.isSidebarExpanded)} - iconOnly={!manager.isSidebarExpanded} + onClick={() => (expanded = !expanded)} + iconOnly={!expanded} btnClasses={'justify-start transition-all duration-150'} title="Conversations" > @@ -45,9 +91,9 @@ unifiedSize="md" variant="subtle" startIcon={{ icon: Plus, classes: 'ml-[2px]' }} - onClick={() => manager.createConversation({ clearMessages: true })} + onClick={newChat} title="Start new conversation" - iconOnly={!manager.isSidebarExpanded} + iconOnly={!expanded} btnClasses={'justify-start transition-all duration-150 whitespace-nowrap'} >
New chat
@@ -56,50 +102,69 @@
- {#if !manager.isSidebarExpanded} + {#if !expanded}
{/if} -
+
+ {#if draftShown && expanded} +
+ +
+ {/if} - {#snippet customRow({ item: conversation, hover })} - {#if manager.isSidebarExpanded} + {#snippet customRow({ item: conversation })} + {#if expanded}
diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index 958f6903bd..bce923c77a 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -151,6 +151,20 @@ export const AI_AGENT_SCHEMA: Schema = { resourceType: 's3object' } }, + // The step's own roster fills `items.enum` in, so the static editor offers the tools this + // agent actually has (`AiAgentStepInputs`). Absence, not an empty list, is what carries every + // tool: a step that holds the field and names nothing has chosen to advertise none. + // Shown for image output as the roster it narrows is, even though neither is used there. + enabled_tools: { + type: 'array', + // Deliberately short. It is the only place the field's text is always on screen rather than + // behind the row's tooltip, and the surface it shows on is the run form, which offers the + // names in a picker and has no unset state to explain. + description: 'Which of the agent tools a run may call.', + items: { + type: 'string' + } + }, max_completion_tokens: { type: 'number', description: 'The most tokens the answer may use.' @@ -178,6 +192,7 @@ export const AI_AGENT_SCHEMA: Schema = { 'memory', 'output_schema', 'user_attachments', + 'enabled_tools', 'max_completion_tokens', 'temperature', 'max_iterations' @@ -291,7 +306,10 @@ export async function loadSchemaFromModule( } return accu }, {}), - schema: AI_AGENT_SCHEMA + // A copy per step, never the shared constant: the form writes back into the property it + // renders (`InputTransformForm` binds `schema.properties[argName]`), and the tool names + // one step offers would otherwise become every step's. + schema: structuredClone(AI_AGENT_SCHEMA) } } diff --git a/frontend/src/lib/components/flows/flowState.ts b/frontend/src/lib/components/flows/flowState.ts index 3123396a9d..595227f57d 100644 --- a/frontend/src/lib/components/flows/flowState.ts +++ b/frontend/src/lib/components/flows/flowState.ts @@ -5,6 +5,7 @@ import { get } from 'svelte/store' import { workspaceStore } from '$lib/stores' import { isFlowModuleTool, agentToolToFlowModule, type AgentTool } from './agentToolUtils' import { linkedToolsScope, setLinkedAgentTools } from './linkedAgentToolsStore.svelte' +import { fetchAgentWithDraft, normalizeAgentRef } from './linkedAgentDrafts' import { loadFlowModuleState } from './flowStateUtils.svelte' import { emptyFlowModuleState } from './utils.svelte' import type { StateStore } from '$lib/utils' @@ -90,7 +91,9 @@ async function mapFlowModule( // the graph can render its tool nodes. They are display-only (their inputs are edited in // the step panel, which infers schemas itself), so no per-tool module state is loaded — // resource tool ids are not flow-unique and must not key into the flow state. - await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id) + // Drafts included: every caller of `initFlowState` is a flow editor, where the graph has + // to show the tools a test would run. Read-only viewers publish for themselves. + await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id, true) } else { // Shape-checked because `tools` is JSON-authored: throwing here would skip the agent's // own state below, leaving it with no schema rather than with no tool schemas. @@ -119,11 +122,17 @@ export async function publishLinkedAgentTools( agentRef: string, workspace: string | undefined, scope: string, - moduleId: string + moduleId: string, + /** Resolve from the agent's unsaved draft when there is one. Editors pass true so the graph + * shows the tool set a test would run; read-only viewers pass false, since a run they are + * displaying used the deployed agent. Required rather than defaulted: an editor call site that + * forgets it republishes the deployed tools over the drafted ones, which reads as the graph + * spontaneously reverting. */ + withDraft: boolean ) { const genKey = `${scope}:${moduleId}` const gen = claimLinkedToolsFetch(scope, moduleId) - const tools = await resolveLinkedAgentTools(agentRef, workspace) + const tools = await resolveLinkedAgentTools(agentRef, workspace, withDraft) if (linkedToolFetchGen.get(genKey) === gen) { setLinkedAgentTools(scope, moduleId, tools, agentRef) } @@ -155,12 +164,27 @@ export function claimLinkedToolsFetch(scope: string, moduleId: string): number { // resource is missing or inaccessible so a broken link never stalls the flow load. export async function resolveLinkedAgentTools( agentRef: string, - workspace?: string + workspace: string | undefined, + withDraft: boolean ): Promise { const ws = workspace ?? get(workspaceStore) if (!ws) return [] - const path = agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') + const path = normalizeAgentRef(agentRef) try { + if (withDraft) { + try { + const { response, draft } = await fetchAgentWithDraft(path, ws) + const value = (draft?.args ?? response.value) as { tools?: AgentTool[] } | undefined + return (value?.tools ?? []) as AgentTool[] + } catch { + // The draft read failed for any reason. This is a display, not a run, so fall through to + // the deployed tools rather than showing an agent with none: an empty node list reads as + // "the agent lost its tools" instead of "we could not reach the server". The paths that + // act on a draft — the previews and the deploy dialog — surface the failure instead. + // Not rethrowing anything here: the outer catch turns every throw into `[]`, so a + // rethrow would skip the very fallback this exists for. + } + } const res = await ResourceService.getResource({ workspace: ws, path }) return ((res.value as { tools?: AgentTool[] } | undefined)?.tools ?? []) as AgentTool[] } catch { diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts new file mode 100644 index 0000000000..fa72aaba5c --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -0,0 +1,181 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + inlineAgentDraft, + inlineAgentDrafts, + loadLinkedAgentDrafts, + repointLinkedAgent, + type LinkedAgentDraft +} from './linkedAgentDrafts' +import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' + +type AiAgentValue = Extract + +function linkedStep(input_transforms: Record): AiAgentValue { + return { + type: 'aiagent', + agent: 'f/team/support', + tool_inputs: { t1: { query: { type: 'javascript', expr: 'flow_input.q' } } }, + input_transforms + } as unknown as AiAgentValue +} + +describe('inlineAgentDraft', () => { + // The overlay order is the worker's: the resource brain first, the step's flow-local inputs on + // top. Reversing it would run the agent author's own `user_message` instead of the flow's. + it('keeps the step wired to the flow while the brain comes from the draft', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }), + { + provider: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' }, + system_prompt: 'answer in french', + user_message: 'the default the agent carries', + tools: [{ id: 't1', summary: 'search' }] + } as any + ) + + expect(inlined.agent).toBeUndefined() + expect(inlined.tools).toEqual([{ id: 't1', summary: 'search' }]) + expect(inlined.input_transforms).toEqual({ + provider: { + type: 'static', + value: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' } + }, + system_prompt: { type: 'static', value: 'answer in french' }, + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }) + // Host bindings are the step's, not the agent's, and the worker overlays them either way. + expect(inlined.tool_inputs).toEqual({ + t1: { query: { type: 'javascript', expr: 'flow_input.q' } } + }) + }) + + // A linked step carries only the flow-local inputs, but one persisted before linking existed can + // still hold stale brain transforms. They must not shadow the draft the test is meant to run. + it('drops brain transforms the step still carries', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'static', value: 'hi' }, + system_prompt: { type: 'static', value: 'stale' } + }), + { system_prompt: 'from the draft' } as any + ) + + expect(inlined.input_transforms).toEqual({ + system_prompt: { type: 'static', value: 'from the draft' }, + user_message: { type: 'static', value: 'hi' } + }) + }) +}) + +describe('inlineAgentDrafts', () => { + // The index is keyed on the bare path while a step may name its agent `$res:`-prefixed, and the + // walk has to reach inside branches and loops. Miss either and every preview silently runs the + // deployed agent — the failure this whole path exists to prevent, and a silent one. + it('reaches a $res:-prefixed link nested in a branch', () => { + const value = { + modules: [ + { + id: 'b', + value: { + type: 'branchone', + default: [], + branches: [ + { + modules: [ + { + id: 'inner', + value: { + type: 'aiagent', + agent: '$res:f/team/support', + tools: [], + input_transforms: { user_message: { type: 'static', value: 'hi' } } + } + } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + const drafts = new Map([ + ['f/team/support', { args: { system_prompt: 'drafted' } } as unknown as LinkedAgentDraft] + ]) + + const inner = (inlineAgentDrafts(value, drafts).modules[0].value as any).branches[0].modules[0] + expect(inner.value.agent).toBeUndefined() + expect(inner.value.input_transforms.system_prompt).toEqual({ + type: 'static', + value: 'drafted' + }) + // The input the flow supplies survives the rewrite. + expect(inner.value.input_transforms.user_message).toEqual({ type: 'static', value: 'hi' }) + }) +}) + +describe('repointLinkedAgent', () => { + // A rename from the agent editor moves every step of the host flow onto the new path, nested ones + // included. Miss one and it silently stays linked to a path that no longer exists. + it('repoints linked steps at any depth and leaves other agents alone', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { + id: 'b', + value: { + type: 'branchall', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { id: 'd', value: { type: 'aiagent', agent: 'f/team/other', tools: [] } } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + expect(repointLinkedAgent(value, 'f/team/support', 'f/team/helpdesk')).toEqual(['a', 'c']) + const branch = (value.modules[1].value as any).branches[0].modules + expect((value.modules[0].value as any).agent).toBe('f/team/helpdesk') + expect(branch[0].value.agent).toBe('f/team/helpdesk') + expect(branch[1].value.agent).toBe('f/team/other') + }) +}) + +// A link the user cannot resolve is an ordinary state and must not block the flow; anything else is +// an outage, and answering "no draft" to one would silently test or deploy against the deployed +// agent while the editor shows the draft. +describe('loadLinkedAgentDrafts error handling', () => { + function failWith(status: number | undefined) { + return async () => { + const err: Error & { status?: number } = new Error('boom') + err.status = status + throw err + } + } + + beforeEach(() => { + vi.restoreAllMocks() + }) + + it.each([401, 403, 404])('treats %i as no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).resolves.toEqual(new Map()) + }) + + it.each([500, undefined])('propagates %s rather than reporting no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).rejects.toThrow( + 'Could not load the agent f/team/support' + ) + }) +}) diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts new file mode 100644 index 0000000000..65624d4429 --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -0,0 +1,248 @@ +import { + ResourceService, + type FlowModule, + type FlowValue, + type InputTransform, + type Resource +} from '$lib/gen' +import { UserDraft } from '$lib/userDraft.svelte' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' +import { dfs } from './dfs' +import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils' +import type { AgentResourceState } from './agentDraft.svelte' +import type { AgentTool } from './agentToolUtils' + +/** A step names its agent bare or as `$res:`/`res://`; all three are the same agent, + * and a draft index has to answer for a lookup written any of those ways. Same normalization as + * `linkedAgentToolsStore`, and as the `trim_start_matches` the worker applies. */ +export function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + +/** Every `ai_agent` resource this flow links to, deduped. `dfs` walks agent tool nodes as well as + * branches and loops, so a nested linked agent tool is included. */ +export function linkedAgentPaths(value: FlowValue | undefined): string[] { + if (!value?.modules) return [] + const paths = new Set() + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent) { + paths.add(normalizeAgentRef(v.agent)) + } + } + return [...paths] +} + +/** Point every step of this flow linked to `from` at `to`, for an agent renamed from inside it. + * Returns the ids of the steps it moved. */ +export function repointLinkedAgent( + value: FlowValue | undefined, + from: string, + to: string +): string[] { + if (!value?.modules) return [] + const moved: string[] = [] + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent === from) { + v.agent = to + moved.push(module.id) + } + } + return moved +} + +/** + * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then + * what a `get_draft` response carried. + * + * Only the live cell is reliably current, and only while an editor holds it: `releaseEntry` drops + * the cached write at refcount 0 on purpose, so once the agent editor closes the persisted row is + * the sole answer. Read through `fetchAgentWithDraft`, which settles that row first. + */ +export function agentDraftState( + response: { draft?: unknown }, + path: string, + workspace: string | undefined +): AgentResourceState | undefined { + const live = UserDraft.get('resource', path, { workspace }) + return live ?? (response.draft as AgentResourceState | undefined) +} + +/** A refusal that already names the agent and says what is wrong with it, so a caller wrapping it + * would only repeat itself. */ +export class AgentDraftUnavailable extends Error {} + +/** + * An agent's resource together with the draft a run of it would use. + * + * The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and + * closing the agent editor releases the in-memory cell without cancelling that pending POST — so + * testing or deploying right after closing would otherwise read a row the last edits have not + * reached yet. `flush` replays the parked save and is a no-op when there is none. + */ +export async function fetchAgentWithDraft( + path: string, + workspace: string +): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + const query = { workspace, itemKind: 'resource' as const, path } + await UserDraftDbSyncer.flush(query) + // `flush` resolves whether or not the save actually landed: `postSave` catches network and + // server errors into its failure map, and answers a conflicting write by parking a snapshot, + // returning normally in both cases. The row about to be read is then older than the edit still + // held in the browser, and nothing downstream could tell. Running that row is a test of the + // wrong agent; deploying it is worse, because the deploy deletes the draft and takes the newer + // edit with it. Neither is recoverable from here, so refuse the read. + const failure = UserDraftDbSyncer.getState(query).failureMessage + if (failure) { + throw new AgentDraftUnavailable(`The unsaved changes to ${path} could not be saved: ${failure}`) + } + if (UserDraftDbSyncer.getConflict(query).conflict) { + throw new AgentDraftUnavailable( + `The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.` + ) + } + const response = await ResourceService.getResource({ workspace, path, getDraft: true }) + return { response, draft: agentDraftState(response, path, workspace) } +} + +/** One linked agent whose resource the user has an unsaved draft for. */ +export interface LinkedAgentDraft { + path: string + /** The draft's resource value: what a run of this agent would use. */ + args: AIAgentConfig + /** The whole draft row, as the resource editors write it — the deploy payload. */ + state: AgentResourceState + /** No deployed row at this path, so deploying has to create rather than update. */ + noDeployed: boolean + /** Of the deployed resource, for `agentDraftCanWrite`. */ + extraPerms: Record +} + +/** Whether `user` may write this agent's resource. Split from the load so that resolving the + * drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it + * looks the user up once for every agent it lists. */ +export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean { + return canWrite(draft.path, draft.extraPerms, user) +} + +/** A link that cannot resolve for the user rather than because something went wrong: the agent was + * deleted, or sits in a folder they cannot read. Both are ordinary states of a rigid link, and + * neither should stop the caller — the flow still tests and deploys, against the deployed agent. + * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy + * the wrong configuration, which is the whole thing this module exists to prevent. */ +export function isExpectedLinkFailure(err: unknown): boolean { + const status = (err as { status?: number } | null | undefined)?.status + return status === 401 || status === 403 || status === 404 +} + +/** + * The unsaved draft of every given `ai_agent` path, for the paths that have one. + * + * Throws when a path fails to load for any reason other than being missing or unreadable, so a + * caller cannot mistake an outage for an agent with nothing unsaved. + */ +export async function loadLinkedAgentDrafts( + paths: string[], + workspace: string | undefined +): Promise> { + const out = new Map() + if (!workspace || paths.length === 0) return out + await Promise.all( + paths.map(async (path) => { + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, workspace)) + } catch (err) { + if (isExpectedLinkFailure(err)) return + if (err instanceof AgentDraftUnavailable) throw err + throw new Error(`Could not load the agent ${path}: ${err}`) + } + if (!draft) return + out.set(path, { + path, + args: (draft.args ?? {}) as AIAgentConfig, + state: draft, + noDeployed: Boolean((response as { no_deployed?: boolean }).no_deployed), + extraPerms: response.extra_perms ?? {} + }) + }) + ) + return out +} + +/** + * Every argument a saved agent carries, as a static input transform. Not only the keys the agent + * form renders: a run reads them all, and an agent holding its own `user_message` answers with it + * when nothing overrides it. `tools` is the step's own roster rather than an input, so it rides on + * the module's `tools` key instead. + */ +export function agentArgsToTransforms(args: AIAgentConfig): Record { + const it: Record = {} + for (const [key, value] of Object.entries(args ?? {})) { + if (key === 'tools' || value === undefined) continue + it[key] = { type: 'static', value } as InputTransform + } + return it +} + +type AiAgentValue = Extract + +/** + * The standalone step a linked step's draft would run as: the draft's brain and tools inlined, with + * the step's own flow-local inputs kept on top. + * + * The overlay order is the worker's (`ai_executor.rs`): its linked branch interpolates the whole + * resource brain and only then writes `user_message`/`user_attachments` back from the step's own + * args. `tool_inputs` stays untouched — the worker overlays it onto the tools in both branches, so + * an inlined step keeps the host flow's tool bindings. + */ +export function inlineAgentDraft(value: AiAgentValue, args: AIAgentConfig): AiAgentValue { + const { agent: _agent, ...rest } = value + return { + ...rest, + tools: (args.tools ?? []) as AgentTool[], + input_transforms: { + ...agentArgsToTransforms(args), + ...flowLocalInputs(value.input_transforms as Record) + } + } as AiAgentValue +} + +/** + * Replace every linked agent step that has a draft with the draft's own configuration, so a preview + * runs what the agent editor is showing rather than the deployed resource. Returns a new value: the + * flow editor hands its live store object to previews. + */ +export function inlineAgentDrafts( + value: FlowValue, + drafts: Map +): FlowValue { + if (drafts.size === 0) return value + // JSON rather than `structuredClone`: the flow editor's value is a Svelte `$state` proxy, which + // `structuredClone` refuses outright. A flow value is JSON by definition — it is about to be + // posted as one — so the round trip loses nothing this preview would have carried. + const next = JSON.parse(JSON.stringify(value)) as FlowValue + for (const module of dfs(next.modules ?? [], (m) => m)) { + const v = module?.value as AiAgentValue | undefined + if (v?.type !== 'aiagent' || !v.agent) continue + const draft = drafts.get(normalizeAgentRef(v.agent)) + if (!draft) continue + module.value = inlineAgentDraft(v, draft.args) + } + return next +} + +/** Load the drafts this flow's linked agents have and inline them. The whole substitution, for a + * caller holding nothing but the value it is about to preview. */ +export async function withAgentDrafts( + value: FlowValue, + workspace: string | undefined +): Promise { + const paths = linkedAgentPaths(value) + if (paths.length === 0) return value + return inlineAgentDrafts(value, await loadLinkedAgentDrafts(paths, workspace)) +} diff --git a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts index ea1674e0b6..29207f58f7 100644 --- a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts +++ b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts @@ -8,7 +8,9 @@ vi.mock('./agentToolUtils', () => ({ isFlowModuleTool: () => false, agentToolToFlowModule: (t: unknown) => t })) -vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } })) +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } +})) import { claimLinkedToolsFetch, @@ -38,8 +40,8 @@ describe('linked tools fetch guard', () => { ) .mockResolvedValueOnce({ value: { tools: [tool('new')] } } as never) - const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') - await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step') + const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) + await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step', false) release?.({ value: { tools: [tool('old')] } }) await stale @@ -56,7 +58,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) setLinkedAgentTools(scope, 'step', [tool('kept')], 'u/admin/a') invalidateLinkedToolsFetches(scope) @@ -73,7 +75,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) claimLinkedToolsFetch(scope, 'step') setLinkedAgentTools(scope, 'step', [tool('direct')], 'u/admin/a') release?.({ value: { tools: [tool('stale')] } }) diff --git a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte index 08399dfdda..d54f8fddb1 100644 --- a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte @@ -128,7 +128,6 @@ {/if}
{:else} - + nid === id ? newId : nid + ) + } + } + } flowStateStore.val[newId] = flowStateStore.val[id] delete flowStateStore.val[id] refreshStateStore(flowStore) @@ -888,7 +898,3 @@ />
- -{#if !disableTutorials} - -{/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte index 234f053f7b..15177d6cd6 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubApps } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubApps() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index 1f3c659bf8..8c07ad1506 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubFlows } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubFlows() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index 207e8612e9..05205d8335 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -4,12 +4,18 @@ import { capitalize } from '$lib/utils' import NoItemFound from '$lib/components/home/NoItemFound.svelte' import { APP_TO_ICON_COMPONENT } from '$lib/components/icons' + import { listHubIntegrationsShared } from '$lib/components/displayNameLoaders' import ListFilters from '$lib/components/home/ListFilters.svelte' - import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' + import { ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { disableHubStore } from '$lib/stores' + import { disableHubStore, workspaceStore } from '$lib/stores' import { logHubScriptPick } from '$lib/utils/featureUsage' + import { + alphabetical, + byPopularity, + localCountsByIntegration + } from '$lib/components/pickerPopularity' interface Props { kind?: HubScriptKind & string @@ -46,19 +52,25 @@ }[] = $state([]) let allApps: string[] = $state([]) + let popularity: (a: string, b: string) => number = $state(alphabetical) let apps: string[] = $derived.by(() => - filter.length > 0 ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() : allApps + filter.length > 0 + ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity) + : allApps ) async function getAllApps(filterKind: typeof kind) { if ($disableHubStore) return try { hubNotAvailable = false - allApps = ( - await IntegrationService.listHubIntegrations({ - kind: filterKind - }) - ).map((x) => x.name) + // Independent reads, so they share one round trip before first paint. + const [integrations, local] = await Promise.all([ + listHubIntegrationsShared(filterKind), + $workspaceStore ? localCountsByIntegration($workspaceStore) : {} + ]) + const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) + popularity = byPopularity(hubPicks, local) + allApps = integrations.map((x) => x.name).sort(popularity) } catch (err) { console.error('Hub is not available') allApps = [] @@ -154,72 +166,73 @@ {#if $disableHubStore} {:else} -
- {@render children?.()} -
- - {#if loading} - - {/if} -
-
- -{#if hubNotAvailable} - - Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. - -{:else if (items.length > 0 && apps.length > 0) || !loading} - - {#if items.length == 0} - - {:else} -
    - {#each items as item (item.path)} -
  • - -
    • - {/each} -
- {/if} - {#if items.length == 20} -
- There are more items than being displayed. Refine your search. +
+ {@render children?.()} +
+ + {#if loading} + + {/if}
+
+ + {#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the + Hub in the instance settings. + + {:else if (items.length > 0 && apps.length > 0) || !loading} + + {#if items.length == 0} + + {:else} +
    + {#each items as item (item.path)} +
  • + +
    • + {/each} +
+ {/if} + {#if items.length == 20} +
+ There are more items than being displayed. Refine your search. +
+ {/if} + {:else} + {#each Array(10).fill(0) as _} + + {/each} {/if} -{:else} - {#each Array(10).fill(0) as _} - - {/each} -{/if} {/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 5204892051..2499c285c4 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -1,9 +1,6 @@ -
- - - {#if $open && active} -
+
+ {#if trigger} + {@render trigger()} + {:else} + + {/if} +
-

{active.description}

- -
    - {#each active.bullets as bullet (bullet)} -
  • - - {bullet} -
    • - {/each} -
- - +
+ {/if} + + +
+ {#snippet rowBody(option: Option, ac: (typeof accentClasses)[string])} +
+ +
+ + {option.label} + + {#if option.badge} + + {option.badge.label} + + {/if} + {/snippet} + {#each allOptions as option (option.key)} + {@const ac = accentClasses[option.accent]} + {@const rowClass = + 'w-full flex flex-row items-center gap-2.5 rounded-md px-2 py-1.5 text-left cursor-pointer transition-colors focus:outline-none data-[highlighted]:bg-surface-hover hover:bg-surface-hover'} + {#if option.variants} + + {#if $wacSubOpen} +
+ {#each option.variants ?? [] as variant (variant.label)} + {@const VariantIcon = variant.icon} + + {/each} +
+ {/if} + {:else} + + {/if} + {/each} + + +
+ + {#if $importSubOpen} +
+ {#each importActions as action, i (action.label)} + + {#if onImportHubProject && i === 0} +
+ {/if} + {/each}
{/if} - -
- {#snippet rowBody(option: Option, ac: (typeof accentClasses)[string])} -
- -
- - {option.label} - - {#if option.badge} - - {option.badge.label} - - {/if} - {/snippet} - {#each allOptions as option (option.key)} - {@const ac = accentClasses[option.accent]} - {@const rowClass = - 'w-full flex flex-row items-center gap-2.5 rounded-md px-2 py-1.5 text-left cursor-pointer transition-colors focus:outline-none data-[highlighted]:bg-surface-hover hover:bg-surface-hover'} - {#if option.variants} - - {#if $wacSubOpen} -
- {#each option.variants ?? [] as variant (variant.label)} - {@const VariantIcon = variant.icon} - - {/each} -
- {/if} - {:else} - - {/if} - {/each} - - -
+ {#if !showDoc} - {#if $importSubOpen} -
- {#each importActions as action (action.label)} - - {/each} -
- {/if} - - {#if !showDoc} - - {/if} -
+ {/if}
- {/if} -
+
+{/if} diff --git a/frontend/src/lib/components/home/HomeAIChat.svelte b/frontend/src/lib/components/home/HomeAIChat.svelte index 441489db65..a8e4d8b973 100644 --- a/frontend/src/lib/components/home/HomeAIChat.svelte +++ b/frontend/src/lib/components/home/HomeAIChat.svelte @@ -46,10 +46,21 @@ import HomeConnectDrawer from './HomeConnectDrawer.svelte' import { USER_SETTINGS_HASH } from '../sidebar/settings' import { prefersSessionHandoff } from '../copilot/chat/global/gate' + import { onboardingProfile } from '$lib/onboardingProfile' const COLLAPSED_SETTING = 'home-ai-composer-collapsed' let value = $state('') + // The stock examples, unless the invite that brought this person here wrote prompts for + // them — those replace the set outright rather than joining it, since a prompt written + // for someone's own stack next to "Ban Discord users" reads as the generic one. + let examples = $state(homeAIExamples) + void onboardingProfile().then((p) => { + if (!p?.starter_prompts?.length) return + examples = p.starter_prompts + promptIndex = 0 + placeholder = examples[0].prompt + }) let placeholder = $state(homeAIExamples[0].prompt) let placeholderVisible = $state(true) let homeConnectDrawer: HomeConnectDrawer | undefined = $state(undefined) @@ -130,7 +141,7 @@ } } - const prompts = homeAIExamples.map((e) => e.prompt) + let prompts = $derived(examples.map((e) => e.prompt)) const CYCLE_MS = 7_000 // Must match the `duration-*` class on the placeholder overlay: the swap happens once the @@ -273,7 +284,7 @@
{#if showComposer && !collapsed}
- {#each homeAIExamples as example (example.label)} + {#each examples as example (example.label)}
+ + + {/snippet} + + {#snippet empty()} +

+ {#if loadFailed} + Could not reach the hub. You can still browse its projects in a new tab. + {:else} + This hub has no projects yet. + {/if} +

+ {/snippet} + + + diff --git a/frontend/src/lib/components/home/ImportProjectModal.svelte b/frontend/src/lib/components/home/ImportProjectModal.svelte new file mode 100644 index 0000000000..d9ac59d3b3 --- /dev/null +++ b/frontend/src/lib/components/home/ImportProjectModal.svelte @@ -0,0 +1,363 @@ + + +{#snippet importPage()} +
+ {#if project} + + + {/if} + + (folder = f)} + onFinish={() => (setup.needed ? (onSetupStep = true) : finish('none'))} + onBack={onClose} + onExecution={(e) => (execution = e)} + resume={execution} + /> +
+{/snippet} + +{#snippet setupPlaceholder()} + +
+ +
+{/snippet} + +{#snippet setupPage()} +
+ finish('skipped', outstanding)} + onFinish={(checked) => finish(checked ? 'filled' : 'unchecked')} + onBack={execution ? () => (onSetupStep = false) : undefined} + /> +
+{/snippet} + + + + {#if slug} + + (onSetupStep = s === 2)} + /> + + { + if (key === IMPORT_PAGE && execution) onSetupStep = false + else if (key === SETUP_PAGE && setup.needed) onSetupStep = true + }} + pages={[ + { key: IMPORT_PAGE, content: importPage }, + { key: SETUP_PAGE, content: setupPage, placeholder: setupPlaceholder } + ]} + /> + {/if} + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 84432112a3..7038a316fa 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -16,7 +16,7 @@ } from '$lib/gen' import { resource } from 'runed' import { getDraftItems } from '$lib/workspaceDrafts.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { disableHubStore, userStore, workspaceStore } from '$lib/stores' import type uFuzzy from '@leeoniya/ufuzzy' import { ArrowDownUp, @@ -40,6 +40,10 @@ type FilterSchemaRec } from '$lib/components/FilterSearchbar.svelte' import NoItemFound from './NoItemFound.svelte' + import WorkspaceEmptyState from './WorkspaceEmptyState.svelte' + import HubProjectPickerModal from './HubProjectPickerModal.svelte' + import ImportProjectModal from './ImportProjectModal.svelte' + import type { HubProjectPick } from '$lib/hubProject' import ListFilters from './ListFilters.svelte' import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' @@ -696,6 +700,10 @@ // runnables an owner holds. A scope change (sort/archive/kind/…) doesn't go // through here: the counts resource keys on those itself. async function reloadItemsAndCounts(): Promise { + // The answer can change with the rows: archiving the last item leaves the listing empty + // with something archived behind it, and a cached "nothing archived" would then call + // the workspace empty and hide the way to it until a page load. + archivedProbe = undefined // A mutated row can be gone, or sit at a new path, afterwards: snapshot what // was on screen so the selection can drop what this reload removes instead of // keeping a dead path. `tick` lets the reloaded rows re-register first. @@ -986,6 +994,105 @@ treeLazyMode && ownerCountsRes.current == undefined && ownerCountsRes.loading ) + // An import just landed, so the rows about to replace the empty state are all new: they + // fade in one after another rather than appearing as a finished list. Cleared on a timer + // because nothing else marks the end — the reload resolves before the rows animate. + let justImported = $state(false) + let justImportedTimer: ReturnType | undefined + function onImported() { + reloadItemsAndCounts() + justImported = true + clearTimeout(justImportedTimer) + justImportedTimer = setTimeout(() => (justImported = false), 2500) + } + + // The hub import, owned here rather than by either entry point: the empty state's link and + // the create menu's Import section open the same dialog, and mounting one per entry point + // would put two of them on the page at once while the workspace is still empty. + let hubPick = $state(undefined) + let hubPickerOpen = $state(false) + + /** + * Whether a workspace the default listing found empty is empty at all, or just has nothing + * unarchived — two different states that want two different things said about them. Asked + * only in that case, and once per workspace: one request for one row, never on a workspace + * with something in it. `hasArchived` is undefined when the request failed — see the catch + * for what that leaves standing. + */ + let archivedProbe = $state<{ workspace: string; hasArchived: boolean | undefined } | undefined>( + undefined + ) + $effect(() => { + const ws = $workspaceStore + if (!ws || !workspaceEmpty || archivedProbe?.workspace === ws) return + untrack(() => void probeArchived(ws)) + }) + async function probeArchived(workspace: string) { + try { + // `includeWithoutMain` to match the listing: the backend drops `auto_kind = 'lib'` + // without it, so a workspace holding only archived library scripts would answer + // "nothing archived". Always true here — hiding library scripts puts a filter in + // `activeFilters`, which `workspaceEmpty` requires to be empty. + const res = await ScriptService.listRunnables({ + workspace, + showArchived: true, + includeWithoutMain: true, + perPage: 1 + }) + archivedProbe = { workspace, hasArchived: (res.items?.length ?? 0) > 0 } + } catch (error) { + // Undefined, not false: false would say the workspace is empty and — since the + // toolbar is inert on the strength of the placeholder carrying the way to archived + // items — leave no way to them at all. Unknown keeps the ordinary caption, which + // promises nothing, and leaves the searchbar live as the fallback it used to be. + console.error('Could not check for archived items:', error) + archivedProbe = { workspace, hasArchived: undefined } + } + } + let emptyStateAnswered = $derived(archivedProbe?.workspace === $workspaceStore) + /** + * The probe could not tell. The toolbar stays usable in that case: `inert` is only right + * while the placeholder is the way to archived items, and here it cannot be. + */ + let archivedUnknown = $derived(emptyStateAnswered && archivedProbe?.hasArchived === undefined) + /** + * Whether this user may be offered the create actions. The empty state's template import + * and create menu do no permission check of their own, so an operator — or a workspace + * whose direct-deploy protection cleared `showEditButtons` — must not be shown them. + * Reading archived items is not a write, so it is not gated on this. + */ + let canCreateHere = $derived(!$userStore?.operator && showEditButtons) + + // The workspace itself holds nothing — no filter is narrowing the list away. It stays + // false until the first load resolves: a skeleton already means "loading", and the + // empty state must not be mistaken for one. The controls it dims stay mounted, so + // nothing moves when the first item lands. + let workspaceEmpty = $derived( + !loading && + !treeCountsPending && + !contentActive && + activeFilters.length === 0 && + filteredItems != undefined && + filteredItems.length === 0 && + visiblePipelineFolders.size === 0 && + !hasMoreServer + ) + /** + * Whether the placeholder below takes the toolbar's job over — it renders under the same + * conditions. Standing the toolbar down depends on something else offering a way onwards: + * where the placeholder holds back, as it does for an operator in a workspace that is + * simply empty, these controls are all there is and stay live. + */ + let placeholderTakesOver = $derived( + workspaceEmpty && emptyStateAnswered && (archivedProbe?.hasArchived === true || canCreateHere) + ) + /** + * The toolbar is dimmed either way; `inert` also takes it off the pointer, which is only + * right while the placeholder carries the way to archived items. A probe that could not + * tell leaves it live as the fallback. + */ + let toolbarInert = $derived(placeholderTakesOver && !archivedUnknown) + // Owners the counts found the user has something in, split by kind. They cover // what the folder/username lists miss: an item shared individually out of a // folder or user space the user is otherwise not a member of. @@ -1651,7 +1758,12 @@ }} > {#if !contentActive} -
+ +
{ @@ -1692,9 +1804,10 @@
{/if} - {#if !loading && !contentActive} + {#if !loading && !contentActive && !workspaceEmpty} + view, expand/collapse (tree only), sort. Nothing to select, group or order on + an empty workspace, so the whole row goes. -->
{#if homeSelection.available && !homeSelection.active}
{/if} - {#if filteredItems?.length == 0} + {#if filteredItems?.length == 0 && !workspaceEmpty}
{/if}
@@ -1820,7 +1941,28 @@ - + {#if workspaceEmpty} + + {#if emptyStateAnswered} + {#if archivedProbe?.hasArchived || canCreateHere} + + (hubPick = project)} + onShowArchived={() => (filterValues.val = { ...filterValues.val, archived: true })} + /> + {:else} + + {/if} + {/if} + {:else} + + {/if} {#if hasMoreServer && !searching} @@ -1849,6 +1991,7 @@ allUsers={treeInjectUsers} ownerCounts={!searching && labelFilter == undefined ? ownerCounts : undefined} selfUsername={$userStore?.username} + groupOtherUsers={treeLazyMode} ownerLoad={treeLazyMode ? ownerLoad : undefined} onExpandOwner={treeLazyMode ? loadOwnerItems : undefined} onCollapseOwner={treeLazyMode ? collapseOwner : undefined} @@ -1863,7 +2006,7 @@ /> {/key} {:else} -
+
{#if filter === ''} {#each [...visiblePipelineFolders].sort() as folder (folder)} {/if} + + (hubPickerOpen = false)} + onPick={(project) => { + hubPickerOpen = false + hubPick = project + }} +/> + (hubPick = undefined)} {onImported} /> + + diff --git a/frontend/src/lib/components/home/ListFilters.svelte b/frontend/src/lib/components/home/ListFilters.svelte index ecddfec11a..6fa2865616 100644 --- a/frontend/src/lib/components/home/ListFilters.svelte +++ b/frontend/src/lib/components/home/ListFilters.svelte @@ -3,6 +3,7 @@ import { Badge } from '../common' import type { BadgeColor, BadgeIconProps } from '../common/badge/model' import { appIconComponent } from '../icons' + import { integrationDisplayName } from '../resourceTypeDisplay' import { onDestroy, onMount } from 'svelte' interface Props { @@ -131,7 +132,7 @@ {/if} - {filter} + {resourceType ? integrationDisplayName(filter) : filter} {#if filter === selectedFilter}✗{/if}
diff --git a/frontend/src/lib/components/home/ListFiltersQuick.svelte b/frontend/src/lib/components/home/ListFiltersQuick.svelte index bca020b482..346fb93492 100644 --- a/frontend/src/lib/components/home/ListFiltersQuick.svelte +++ b/frontend/src/lib/components/home/ListFiltersQuick.svelte @@ -1,6 +1,7 @@ +{#snippet ownerNode(node: RootNode, loadRank: number, indent: number)} + +{/snippet} + {#if groupedItems === 'loading'}
{:else}
- {#each groupedItems.slice(0, nbDisplayed) as item, rootIndex ('folderName' in item ? `f__${item.folderName}` : 'username' in item ? `u__${item.username}` : `i__${item.type}__${item.path}`)} - {#if item} - + {#each rows.slice(0, nbDisplayed) as row (row.kind === 'otherUsers' ? 'other_users' : 'folderName' in row.node ? `f__${row.node.folderName}` : 'username' in row.node ? `u__${row.node.username}` : `i__${row.node.type}__${row.node.path}`)} + {#if row.kind === 'otherUsers'} + + + +
+
+ +
+ Other users +
+ ({pluralize(otherUsers.length, 'user')}{otherUsersItemCount != undefined + ? ` · ${pluralize(otherUsersItemCount, 'item')}` + : ''}) +
+
+
+
+ {#if otherUsersOpen} + + {#each otherUsers.slice(0, nbOtherUsersDisplayed) as user, i (user.username)} + {@render ownerNode(user, ownerRowCount + i, 1)} + {/each} + {#if nbOtherUsersDisplayed < otherUsers.length} +
+ + Showing {nbOtherUsersDisplayed} of {otherUsers.length} users + + +
+ {/if} + {/if} + {:else} + {@render ownerNode(row.node, row.loadRank, 0)} {/if} {/each} - {#if nbDisplayed < groupedItems.length || hasMoreServer} + {#if nbDisplayed < rows.length || hasMoreServer}
- {#if nbDisplayed < groupedItems.length} - Showing {nbDisplayed} of {groupedItems.length} folders and users + {#if nbDisplayed < rows.length} + Showing {shownOwnerRowCount} of {ownerRowCount} folders and users {:else} @@ -231,12 +355,12 @@ unifiedSize="sm" variant="subtle" on:click={() => { - if (nbDisplayed < groupedItems.length) - nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, groupedItems.length) + if (nbDisplayed < rows.length) + nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, rows.length) else onLoadMore?.() }} > - {nbDisplayed < groupedItems.length ? 'Show more' : 'Load more'} + {nbDisplayed < rows.length ? 'Show more' : 'Load more'}
{/if} diff --git a/frontend/src/lib/components/home/TutorialBanner.svelte b/frontend/src/lib/components/home/TutorialBanner.svelte deleted file mode 100644 index ef7b30c27e..0000000000 --- a/frontend/src/lib/components/home/TutorialBanner.svelte +++ /dev/null @@ -1,178 +0,0 @@ - - -{#if !isDismissed} - -
- - {#if hasCompletedAny} - New tutorial available! - {:else} - First time? - {/if} - - - -
-{/if} diff --git a/frontend/src/lib/components/home/TutorialButton.svelte b/frontend/src/lib/components/home/TutorialButton.svelte deleted file mode 100644 index 6a31ae9569..0000000000 --- a/frontend/src/lib/components/home/TutorialButton.svelte +++ /dev/null @@ -1,124 +0,0 @@ - - - - diff --git a/frontend/src/lib/components/home/WorkspaceEmptyState.svelte b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte new file mode 100644 index 0000000000..fc36cc334b --- /dev/null +++ b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte @@ -0,0 +1,146 @@ + + +
+ {#each rowOpacities as opacity, i (i)} + + {/each} + + +
+ {#if archivedOnly} + + + Everything in this workspace is archived. + . + + {:else} + Your scripts, flows and apps will show up here. + {/if} + {#if canCreate} + + {#if !$disableHubStore} + + + e.detail && logFeatureUsage('home', 'template_picker_open', { key: 'empty_state' })} + > + {#snippet trigger()}Start from a template{/snippet} + {#snippet content({ close })} + { + close() + onPick(project) + }} + /> + {/snippet} + + or + {/if} + + {#snippet trigger()} + + . + {/snippet} + + {/if} +
+
diff --git a/frontend/src/lib/components/icons/BRAND_COLORS.md b/frontend/src/lib/components/icons/BRAND_COLORS.md index 1cff77a3d3..11d4df5c3b 100644 --- a/frontend/src/lib/components/icons/BRAND_COLORS.md +++ b/frontend/src/lib/components/icons/BRAND_COLORS.md @@ -179,6 +179,7 @@ repeatedly — check the brand's own page. | `KeycloakIcon` | `keycloak` | fixed | #00B8E3 | #00B8E3 | 8.18 | 10.65 | keycloak.org's own mark, https://www.keycloak.org/resources/images/icon.svg (cyan #00B8E3/#33C6E9/#008AAA over greys #4D4D4D–#EDEDED, single theme) | | `KlaviyoIcon` | `klaviyo` | pair | #1D1E20 | #FFFFFF | 16.14 | 12.47 | klaviyo.com --color-core-charcoal; the flag mark is the standalone logomark the site header collapses to, and the shape of klaviyo.com/icons/icon-512x512.png | | `KoboToolboxIcon` | `kobotoolbox` | fixed | #2095F3 | #2095F3 | 3.05 | 3.95 | the kobotoolbox.org header logo and $kobo-blue in kobotoolbox/kpi jsapp/scss/colors.scss | +| `KubernetesIcon` | — | fixed | #326CE5 | #326CE5 | 4.61 | 2.62 | the CNCF artwork repo (github.com/cncf/artwork/projects/kubernetes/icon/color/kubernetes-icon-color.svg, CC-BY-4.0) | | `KustomerIcon` | `kustomer` | fixed | #FBEC2A | #FBEC2A | 14.08 | 12.47 | kustomer.com/images/kustomer/Kusty.svg | | `LangfuseIcon` | `langfuse` | fixed | #FF5D5F | #FF5D5F | 2.91 | 4.47 | langfuse.com/brand "Icon - Color (SVG)", used unmodified | | `LessIcon` | — | pair | #274F82 | #FFFFFF | 8.04 | 12.47 | github.com/less/logo (MIT) | @@ -443,6 +444,7 @@ Constraints that would otherwise be broken by a well-meaning change. - **JsonIcon** — JSON itself has no brand owner or published colours — json.org states none — so this is a Material palette pick, not a brand colour. - **KanidmIcon** — Kanidm's artwork is CC-BY-NC-ND — no recolouring or other derivatives. - **KlaviyoIcon** — Klaviyo draws it in currentColor, hence the white swap on dark. +- **KubernetesIcon** — The Linux Foundation trademark guidelines allow only the published variants (colour, all-blue, black, white); the colour mark's white knockouts are part of the artwork, not a theme swap. - **LangfuseIcon** — Langfuse's trademark terms forbid modifying the assets. - **LineIcon** — LINE forbids any change to the logo's colour, so there is no reversed variant. - **LinearIcon** — Guidelines ship a light/dark logomark pair and forbid altering the assets in any other way. @@ -571,7 +573,7 @@ Not brands. These inherit `currentColor` on purpose and must not be given a pair ## Coverage -- brand icons: **314**, of which **310** carry a recorded source +- brand icons: **315**, of which **311** carry a recorded source - per-theme pairs applied: **136** (5 of them by inversion or a two-SVG swap, see above) - concept icons: **34** - effectively invisible on light: **1** (AbstractApiIcon) diff --git a/frontend/src/lib/components/icons/KubernetesIcon.svelte b/frontend/src/lib/components/icons/KubernetesIcon.svelte new file mode 100644 index 0000000000..76279e002b --- /dev/null +++ b/frontend/src/lib/components/icons/KubernetesIcon.svelte @@ -0,0 +1,35 @@ + + + + + + + + + diff --git a/frontend/src/lib/components/instanceBanner.test.ts b/frontend/src/lib/components/instanceBanner.test.ts new file mode 100644 index 0000000000..bc93af18a1 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from 'vitest' +import { + INSTANCE_BANNER_MESSAGE_MAX_LEN, + isInstanceBannerVisible, + resolveInstanceBanner +} from './instanceBanner' + +const BANNER = { + enabled: true, + message: 'Scheduled maintenance on Saturday.', + severity: 'warning', + dismissible: true +} + +describe('resolveInstanceBanner', () => { + it('drops a link that is not an absolute http(s) URL', () => { + // Declarative instance config writes global_settings rows directly, so the API's + // scheme check is not the only thing standing between a stored value and an href. + for (const link of ['javascript:alert(1)', 'data:text/html,x', 'status.example.com', 123]) { + expect(resolveInstanceBanner({ ...BANNER, link })?.link).toBeUndefined() + } + expect(resolveInstanceBanner({ ...BANNER, link: 'https://status.example.com' })?.link).toBe( + 'https://status.example.com' + ) + }) + + it('keeps a message the backend accepted whole', () => { + // The backend caps at INSTANCE_BANNER_MESSAGE_MAX_LEN code points (`chars().count()`). + // Truncating with `slice` here would count UTF-16 units and halve an all-emoji message + // that passed validation, so the two sides must measure the same way. + const emoji = '\u{1F6A7}'.repeat(INSTANCE_BANNER_MESSAGE_MAX_LEN) + expect([...resolveInstanceBanner({ ...BANNER, message: emoji })!.message]).toHaveLength( + INSTANCE_BANNER_MESSAGE_MAX_LEN + ) + }) + + it('shows nothing when disabled or without a message', () => { + // An enabled banner with no message is a writable state (the backend accepts it so a + // half-typed announcement cannot fail an admin's whole settings save), so this is the + // only thing keeping it off everyone's screen. + expect(resolveInstanceBanner({ ...BANNER, enabled: false })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: ' ' })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: 42 })).toBeUndefined() + }) +}) + +describe('isInstanceBannerVisible', () => { + it('honours a dismissal only while the announcement is dismissible', () => { + const dismissible = resolveInstanceBanner(BANNER)! + expect(isInstanceBannerVisible(dismissible, dismissible.fingerprint)).toBe(false) + expect(isInstanceBannerVisible(dismissible, 'some other announcement')).toBe(true) + + // Escalating the same announcement to mandatory must reach the people who already + // dismissed it — the fingerprint does not change, so nothing else would bring it back. + const mandatory = resolveInstanceBanner({ ...BANNER, dismissible: false })! + expect(mandatory.fingerprint).toBe(dismissible.fingerprint) + expect(isInstanceBannerVisible(mandatory, mandatory.fingerprint)).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/instanceBanner.ts b/frontend/src/lib/components/instanceBanner.ts new file mode 100644 index 0000000000..f1693024b8 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.ts @@ -0,0 +1,129 @@ +import { SettingService } from '$lib/gen' + +/** Drives the banner palette and icon; the subset of `AlertType` that fits an announcement. */ +export type InstanceBannerSeverity = 'info' | 'warning' | 'error' + +/** Stored shape of the `instance_banner` global setting. Every field is optional: a + * stored value can predate a field this code knows about. */ +export interface InstanceBanner { + enabled?: boolean + message?: string + severity?: InstanceBannerSeverity + /** Whether a viewer may dismiss the banner for themselves. Absent means yes. */ + dismissible?: boolean + link?: string + link_label?: string +} + +export const INSTANCE_BANNER_SETTING = 'instance_banner' + +/** Mirror `INSTANCE_BANNER_MESSAGE_MAX_LEN` / `INSTANCE_BANNER_LINK_LABEL_MAX_LEN` in + * backend/windmill-common/src/global_settings.rs, which reject longer values at write time. */ +export const INSTANCE_BANNER_MESSAGE_MAX_LEN = 500 +export const INSTANCE_BANNER_LINK_LABEL_MAX_LEN = 60 + +export type ResolvedInstanceBanner = { + message: string + severity: InstanceBannerSeverity + dismissible: boolean + link?: string + linkLabel: string + /** Dismissal token: a viewer who dismissed one announcement sees the next one, + * because editing any displayed part of the banner changes this string. */ + fingerprint: string +} + +/** + * Read a stored banner field as a string. + * + * The setting is a raw `global_settings` row, so its shape is only ever as good as the + * writer that last touched it — and anything here that calls `.trim()` on a number throws, + * taking the whole settings form down with it. + */ +export function bannerString(value: unknown): string { + return typeof value === 'string' ? value : '' +} + +/** + * Truncate to `max` code points, matching the backend's `chars().count()` cap. + * + * `String.slice` counts UTF-16 code units, so it would cut a 500-emoji message the backend + * accepted in half — and the two sides must agree on what "500 characters" means. + */ +function truncateChars(value: string, max: number): string { + const chars = [...value] + return chars.length > max ? chars.slice(0, max).join('') : value +} + +export function isHttpUrl(value: string): boolean { + try { + const url = new URL(value) + return url.protocol === 'http:' || url.protocol === 'https:' + } catch { + return false + } +} + +/** + * Turn the raw setting into what the banner renders, or `undefined` for "show nothing". + * + * The scheme check repeats the one the writers run on purpose. The link becomes the href of + * an anchor shown to every user of the instance, and this is the last place that can refuse + * it — a row predating the validator, or written straight to the table, reaches here having + * passed nothing. + */ +export function resolveInstanceBanner(raw: unknown): ResolvedInstanceBanner | undefined { + if (!raw || typeof raw !== 'object') return undefined + const banner = raw as InstanceBanner + const message = bannerString(banner.message).trim() + if (banner.enabled !== true || message === '') return undefined + + const severity: InstanceBannerSeverity = + banner.severity === 'warning' || banner.severity === 'error' ? banner.severity : 'info' + const rawLink = bannerString(banner.link).trim() + const link = isHttpUrl(rawLink) ? rawLink : undefined + const linkLabel = + truncateChars(bannerString(banner.link_label).trim(), INSTANCE_BANNER_LINK_LABEL_MAX_LEN) || + 'Learn more' + + return { + message: truncateChars(message, INSTANCE_BANNER_MESSAGE_MAX_LEN), + severity, + dismissible: banner.dismissible !== false, + link, + linkLabel, + fingerprint: JSON.stringify([message, severity, link ?? '', link ? linkLabel : '']) + } +} + +/** + * The reason the banner form cannot be saved, or `undefined` when it can. + * + * Shared with the setting's `isValid` so the Save button and the inline message agree: the + * backend refuses a bad link, and a category save fires its settings concurrently, so a Save + * that got this far would persist the other Core settings and fail only the banner. + */ +export function instanceBannerFormError(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const link = bannerString((value as InstanceBanner).link).trim() + return link !== '' && !isHttpUrl(link) ? 'Link must be an absolute http(s) URL' : undefined +} + +/** + * Whether a viewer holding `dismissedFingerprint` should see this announcement. + * + * A non-dismissible announcement ignores stored dismissals entirely: an admin escalating an + * existing notice to mandatory must reach the people who already dismissed it, and the + * fingerprint deliberately does not cover `dismissible`, so nothing else would bring it back. + */ +export function isInstanceBannerVisible( + banner: ResolvedInstanceBanner | undefined, + dismissedFingerprint: string +): banner is ResolvedInstanceBanner { + if (banner == undefined) return false + return !banner.dismissible || dismissedFingerprint !== banner.fingerprint +} + +export async function fetchInstanceBanner(): Promise { + return resolveInstanceBanner(await SettingService.getGlobal({ key: INSTANCE_BANNER_SETTING })) +} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 8bbca5b4be..34079eae51 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -1,5 +1,7 @@ import type { ButtonType } from './common/button/model' +import { allowedOriginsSettingError } from './triggers/http/utils' import { z } from 'zod' +import { instanceBannerFormError } from './instanceBanner' import { writable } from 'svelte/store' /** @@ -68,6 +70,7 @@ export interface Setting { | 'webhook_base_url' | 'ws_connectivity' | 'retention_overrides' + | 'instance_banner' storage: SettingStorage advancedToggle?: { label: string @@ -236,6 +239,19 @@ export const settings: Record = { placeholder: 'only for EE', storage: 'setting' }, + { + label: 'Announcement banner', + description: + 'Message shown above every page of the instance, for maintenance windows and incidents.', + key: 'instance_banner', + fieldType: 'instance_banner', + storage: 'setting', + // The banner only renders on the managed cloud, so only offer it there. + cloudonly: true, + hideInQuickSetup: true, + // Gates Save. The card renders the specific message itself, so no `error` here. + isValid: (value: any) => instanceBannerFormError(value) == undefined + }, { label: 'Non-prod instance', description: @@ -266,6 +282,22 @@ export const settings: Record = { ee_only: '', hideInQuickSetup: true }, + { + label: 'HTTP route default allowed origins', + description: + 'Origins that HTTP routes allow to call them from a browser when the route sets none of its own. A route overrides this with its own list, and opts out entirely by setting its allowed origins to *. Leave unset for no instance-wide default, so every route is callable from any origin unless it restricts itself.', + key: 'http_route_default_allowed_origins', + fieldType: 'text', + placeholder: 'https://app.example.com, https://admin.example.com', + storage: 'setting', + error: + 'Each origin must be visible ASCII with no comma, and there can be at most 100 of them. null is not allowed, since every sandboxed iframe sends it.', + // The same check the API applies, so a value it would refuse cannot be + // saved here and then silently drop to no restriction at the next boot. + isValid: (value: unknown) => allowedOriginsSettingError(value) === undefined, + ee_only: '', + hideInQuickSetup: true + }, { label: 'Audit log retention (days)', key: 'audit_log_retention_days', @@ -531,6 +563,17 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, + { + label: 'Back AI sessions up to the instance object storage', + description: + "Browsers back their AI sessions up to their workspace's object storage, encrypted with the workspace key. When this is on and instance object storage is configured, a workspace without object storage of its own uses the instance object storage instead, under the same encryption; configuring a storage for the workspace moves its backups there and deletes what it kept in the instance storage. On by default; turn off to keep the AI sessions of such workspaces in the browser only.", + key: 'ai_sessions_instance_storage_fallback', + fieldType: 'boolean', + defaultValue: () => true, + storage: 'setting', + ee_only: '', + hideInQuickSetup: true + }, { label: 'Store audit logs in object storage', description: diff --git a/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte index 98fb15c4bd..d55c2f9ce6 100644 --- a/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte +++ b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte @@ -65,33 +65,46 @@ description="People your identity provider authenticated who opened an app set to Guests without a Windmill account. One email is one guest, however many workspaces it opened." /> -
- {#key usage} - setInstanceSwitch(e.detail)} - options={{ - right: 'Allow guests on this instance', - rightTooltip: - 'Off, no guest can sign in anywhere, whatever a workspace or an app says, and sessions already issued stop on their next request.' - }} - /> - {/key} -
+{#if !usage.available} +
+ + No guest can sign in here, whatever a workspace or an app says. Guests require a self-hosted + instance or a dedicated Windmill Cloud deployment. + +
+{:else} +
+ {#key usage} + setInstanceSwitch(e.detail)} + options={{ + right: 'Allow guests on this instance', + rightTooltip: + 'Off, no guest can sign in anywhere, whatever a workspace or an app says, and sessions already issued stop on their next request.' + }} + /> + {/key} +
-
- - {#if usage.metered} - Beyond the allowance, every four guests count as one seat{usage.guest_seats > 0 - ? `: ${usage.billable_guests} guests past it take ${usage.guest_seats} ${usage.guest_seats === 1 ? 'seat' : 'seats'} now` - : ''}. - {:else} - Beyond the allowance, new guests are refused until the count drops below it; an - Enterprise license meters them instead. - {/if} - -
+
+ + {#if usage.metered} + Beyond the allowance, every four guests count as one seat{usage.guest_seats > 0 + ? `: ${usage.billable_guests} guests past it take ${usage.guest_seats} ${usage.guest_seats === 1 ? 'seat' : 'seats'} now` + : ''}. + {:else} + Beyond the allowance, new guests are refused until the count drops below it; an Enterprise + license meters them instead. + {/if} + +
+{/if} + import type { Writable } from 'svelte/store' + import Toggle from '../Toggle.svelte' + import TextInput from '../text_input/TextInput.svelte' + import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' + import { classes as alertClasses, icons as alertIcons } from '../common/alert/model' + import { + bannerString, + INSTANCE_BANNER_LINK_LABEL_MAX_LEN, + INSTANCE_BANNER_MESSAGE_MAX_LEN, + INSTANCE_BANNER_SETTING, + instanceBannerFormError, + resolveInstanceBanner, + type InstanceBanner, + type InstanceBannerSeverity + } from '../instanceBanner' + + interface Props { + values: Writable> + disabled?: boolean + } + + let { values, disabled = false }: Props = $props() + + // Writes go through `$values[...]` member assignments: only those push the nested + // mutation back into the store, which is what the unsaved-changes check and the Save + // button watch. Reads go through a *snapshot* because the form mutates that object in + // place — a derived returning the object itself keeps its identity across edits, and + // Svelte then stops propagating to whatever depends on it. Every getter below applies + // the same default `resolveInstanceBanner` does, so a value missing a field displays + // as it will actually render. + let banner: InstanceBanner = $derived( + $state.snapshot($values[INSTANCE_BANNER_SETTING] ?? {}) as InstanceBanner + ) + + const severities: { value: InstanceBannerSeverity; label: string }[] = [ + { value: 'info', label: 'Info' }, + { value: 'warning', label: 'Warning' }, + { value: 'error', label: 'Critical' } + ] + + // Runs the resolver the banner itself uses, so this shows what the instance gets — + // including the "nothing is shown" cases (disabled, or an empty message). + let preview = $derived(resolveInstanceBanner(banner)) + let linkError = $derived(instanceBannerFormError(banner)) + + +
+ banner.enabled === true, (v) => ($values[INSTANCE_BANNER_SETTING].enabled = v) + } + options={{ right: 'Show the banner to every user' }} + /> + +
+ Message + bannerString(banner.message), + (v) => ($values[INSTANCE_BANNER_SETTING].message = String(v)) + } + /> +
+ +
+ Severity + bannerString(banner.severity) || 'info', + (v) => ($values[INSTANCE_BANNER_SETTING].severity = v) + } + > + {#snippet children({ item })} + {#each severities as severity (severity.value)} + + {/each} + {/snippet} + +
+ +
+ Link (optional) +
+ bannerString(banner.link), + (v) => ($values[INSTANCE_BANNER_SETTING].link = String(v)) + } + /> + bannerString(banner.link_label), + (v) => ($values[INSTANCE_BANNER_SETTING].link_label = String(v)) + } + /> +
+ {#if linkError} + {linkError} + {/if} +
+ + banner.dismissible !== false, (v) => ($values[INSTANCE_BANNER_SETTING].dismissible = v) + } + options={{ + right: 'Let users dismiss it', + rightTooltip: + 'Dismissal is remembered per browser and only for this exact announcement: editing the message, severity or link brings it back for everyone. Turning this off also shows it again to everyone who had dismissed it.' + }} + /> + +
+ Preview + {#if preview} + {@const palette = alertClasses[preview.severity]} + {@const Icon = alertIcons[preview.severity]} +
+ + {preview.message} + {#if preview.link} + {preview.linkLabel} + {/if} +
+ {:else} + + {banner.enabled === true + ? 'Nothing is shown until the message is filled in.' + : 'Nothing is shown while the banner is off.'} + + {/if} +
+
diff --git a/frontend/src/lib/components/job_args.test.ts b/frontend/src/lib/components/job_args.test.ts new file mode 100644 index 0000000000..cf5818ff0f --- /dev/null +++ b/frontend/src/lib/components/job_args.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from 'vitest' +import { + coerceArgsToSchema, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs +} from './job_args' + +describe('coerceArgsToSchema', () => { + // A scalar widget renders its own reading of a wrong-typed value and never writes that + // reading back, so an untouched form submits something it never displayed: a number + // input paints `"12"` as a filled-looking 12, and a toggle shows `"false"` as on. + it('converts a value its widget would read, so the form shows what runs', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' }, + name: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: '12', flag: 'false', label: 3, name: 'ada' }, + schema + ) + expect(args).toEqual({ count: 12, flag: false, label: '3', name: 'ada' }) + expect(clearedKeys).toEqual([]) + }) + + // Cleared, not carried: the widget shows nothing for these, so nothing is what an + // untouched form should send. + it('empties a value with no reading in its declared type', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: 'abc', flag: 'maybe', label: { a: 1 } }, + schema + ) + expect(args).toEqual({}) + expect(clearedKeys.sort()).toEqual(['count', 'flag', 'label']) + }) + + // A field the schema does not name is a field no run surface in the product draws, so a + // value under that name would reach the job without anyone having been able to see it. + // `constructor` is declared by every object through its prototype and by no schema. + it('drops arguments the schema does not declare, naming them', () => { + const kept = coerceArgsToSchema({ a: 'keep', b: 2, constructor: 'x' }, { + properties: { a: { type: 'string' } } + } as any) + expect(kept.args).toEqual({ a: 'keep' }) + expect(kept.undeclaredKeys).toEqual(['b', 'constructor']) + // Declaring nothing is declaring no arguments, which is what a `**kwargs` script and a + // schema that failed to infer both look like. + expect(coerceArgsToSchema({ a: 1 }, undefined).args).toEqual({}) + }) + + // Resolved by the job, so the declared type describes what it receives and never the + // string standing in for it. `Number('$var:…')` is NaN, so coercing would destroy it. + it('leaves a variable or resource reference in any slot', () => { + const schema = { + properties: { + size: { type: 'number' }, + on: { type: 'boolean' }, + db: { type: 'object', format: 'resource-postgresql' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { size: '$var:u/admin/size', on: '$var:u/admin/on', db: '$res:u/admin/pg' }, + schema + ) + expect(args).toEqual({ + size: '$var:u/admin/size', + on: '$var:u/admin/on', + db: '$res:u/admin/pg' + }) + expect(clearedKeys).toEqual([]) + }) + + // Not merely unreadable: `MultiSelect` maps over the value as it renders, so anything + // else throws and takes the whole card down, Cancel with it. A reference is no + // exception — the widget draws before anything resolves — so this slot is the one + // place the reference rule above does not hold. + it('empties a non-array in a dyn-multiselect slot, reference included', () => { + const schema = { properties: { tags: { type: 'object', format: 'dynmultiselect-list' } } } + expect(coerceArgsToSchema({ tags: ['a'] }, schema).args).toEqual({ tags: ['a'] }) + for (const bad of [{ a: 1 }, '$var:u/admin/watchlist']) { + const { args, clearedKeys } = coerceArgsToSchema({ tags: bad }, schema) + expect(args).toEqual({}) + expect(clearedKeys).toEqual(['tags']) + } + }) + + // Below the top the form has the same limitations as everywhere else in the product, + // and descending means resolving `oneOf` branches — where being wrong rewrites what the + // user typed into the branch they did open. + it('leaves nested and container values to the widget that renders them', () => { + const schema = { + properties: { + obj: { type: 'object', properties: { known: { type: 'string' } } }, + rows: { type: 'array', items: { type: 'object' } } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }, + schema + ) + expect(args).toEqual({ obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }) + expect(clearedKeys).toEqual([]) + }) + + // Both sides parsed, never written as literals: `__proto__:` in an object literal is + // the prototype setter, so a literal declares nothing to coerce in the first place. + it('keeps a declared __proto__ instead of losing it to the setter', () => { + const { args } = coerceArgsToSchema( + JSON.parse('{"__proto__":"legit","keep":1}'), + JSON.parse('{"properties":{"__proto__":{"type":"string"},"keep":{"type":"number"}}}') + ) + expect(Object.hasOwn(args, '__proto__')).toBe(true) + expect(args['__proto__']).toBe('legit') + }) +}) + +describe('enforceDisabledDefaults', () => { + const schema = { + properties: { + locked: { type: 'string', disabled: true, default: 'fixed' }, + open: { type: 'string' } + } + } + + it('overwrites a disabled field and reports only what it changed', () => { + expect(enforceDisabledDefaults({ locked: 'mine', open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: ['locked'] + }) + // Never supplied is not overwritten: the field shows the default either way, and a + // caller told otherwise would try to correct what it never sent. + expect(enforceDisabledDefaults({ open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: [] + }) + }) + + it('reports no reset for an object default the caller already matched', () => { + const objSchema = { + properties: { conf: { type: 'object', disabled: true, default: { a: 1 } } } + } + expect(enforceDisabledDefaults({ conf: { a: 1 } }, objSchema).resetKeys).toEqual([]) + }) +}) + +describe('secret args at every level the form nests', () => { + const schema = { + properties: { + top: { type: 'string', password: true }, + obj: { properties: { inner: { type: 'string', password: true } } }, + list: { items: { properties: { secret: { type: 'string', password: true } } } }, + either: { + oneOf: [ + { title: 'a', properties: { key: { type: 'string', password: true } } }, + { title: 'b', properties: { other: { type: 'string', password: true } } } + ] + } + } + } + const args = { + top: 'hunter2', + obj: { inner: '$var:u/ada/prod', keep: 1 }, + list: [{ secret: 'one', name: 'a' }, { secret: 'two' }], + // Tagged as branch 'a', but 'b' is stripped too: the tag is runtime state. + either: { kind: 'a', key: 'k', other: 'o' } + } + + it('redacts every value and keeps every reference', () => { + const redacted = JSON.stringify(redactSecretArgs(args, schema)) + for (const secret of ['hunter2', 'one', 'two', '"k"', '"o"']) { + expect(redacted).not.toContain(secret) + } + expect(redacted).toContain('') + expect(redacted).toContain('"name":"a"') + expect(redacted).toContain('$var:u/ada/prod') + }) + + // ArgInput synthesises '' for every untouched string, so marking one would put a hidden + // value on the card for a field nobody filled in — and mint nothing to back it. + it('leaves an empty secret empty', () => { + expect( + redactSecretArgs({ tok: '' }, { properties: { tok: { type: 'string', password: true } } }) + ).toEqual({ tok: '' }) + }) + + it('reaches a secret under a oneOf branch of an array element', () => { + const oneOfItems = { + properties: { + steps: { + type: 'array', + items: { + oneOf: [{ title: 'push', properties: { token: { type: 'string', password: true } } }] + } + } + } + } + expect(redactSecretArgs({ steps: [{ token: 'hunter2', name: 'a' }] }, oneOfItems)).toEqual({ + steps: [{ token: '', name: 'a' }] + }) + }) + + // The walk descends on the value's shape: routing an array down `properties` because the + // declaration carries that key would visit none of its elements, leaving the secret in + // the persisted card verbatim. + it('reaches through a declaration carrying both items and properties', () => { + const both = { + properties: { + creds: { + type: 'array', + items: { properties: { token: { type: 'string', password: true } } }, + properties: { token: { type: 'string', password: true } } + } + } + } + expect(redactSecretArgs({ creds: [{ token: 'hunter2' }] }, both)).toEqual({ + creds: [{ token: '' }] + }) + }) + + // A container shaped unlike its declaration is kept, so the walk has to reach in through + // the half the declaration does carry — on the value's shape alone it stops at the + // mismatch, leaving the secret there for the persisted card and the model to read. + it('reaches through a container shaped unlike its declaration', () => { + const declaresArray = { + properties: { + rows: { type: 'array', items: { properties: { token: { password: true } } } } + } + } + expect(redactSecretArgs({ rows: { token: 'hunter2' } }, declaresArray)).toEqual({ + rows: { token: '' } + }) + + const declaresObject = { + properties: { cfg: { type: 'object', properties: { token: { password: true } } } } + } + expect(redactSecretArgs({ cfg: [{ token: 'hunter2' }] }, declaresObject)).toEqual({ + cfg: [{ token: '' }] + }) + }) +}) + +describe('redactFileArgs', () => { + const schema = { + properties: { + doc: { type: 'string', contentEncoding: 'base64' }, + pics: { type: 'array', items: { type: 'string', contentEncoding: 'base64' } }, + wrap: { properties: { inner: { type: 'string', contentEncoding: 'base64' } } }, + note: { type: 'string' } + } + } + + it('replaces the bytes with a size marker at every level, and keeps the rest', () => { + const oneMeg = 'A'.repeat(1024 * 1024 * 2) + const redacted = redactFileArgs( + { doc: oneMeg, pics: ['B'.repeat(4096)], wrap: { inner: 'C'.repeat(2048) }, note: 'hi' }, + schema + ) + expect(redacted).toEqual({ + doc: '', + pics: [''], + wrap: { inner: '' }, + note: 'hi' + }) + }) +}) diff --git a/frontend/src/lib/components/job_args.ts b/frontend/src/lib/components/job_args.ts index ef3377081f..9698c03cd4 100644 --- a/frontend/src/lib/components/job_args.ts +++ b/frontend/src/lib/components/job_args.ts @@ -1,5 +1,303 @@ +/** + * A job's arguments prepared for a run form, its readers, and a result view. Coercing must + * not lose what the caller meant to send, so it is exact and shallow; stripping and + * redacting only blank a field, so they go to any depth and err towards visiting too much. + */ import { deepEqual } from 'fast-equals' +const isLockedProp = (prop: any) => !!prop?.disabled && 'default' in prop + +/** + * A field the schema disables is not the caller's to set: the run sends the schema's + * default whatever it holds. Top-level only, like every filter here. Returns the keys it + * overwrote; notifying is the caller's job. + */ +export function enforceDisabledDefaults( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; resetKeys: string[] } { + // Null prototype: assigning a declared `__proto__` into a plain `{}` reaches the + // inherited setter and the default vanishes. Always copied — callers bind the result to + // a form that edits in place, so returning the input would write through to theirs. + const result: Record = Object.assign(Object.create(null), args) + if (!schema?.properties) return { args: { ...result }, resetKeys: [] } + const resetKeys: string[] = [] + for (const [key, prop] of Object.entries(schema.properties)) { + if (!isLockedProp(prop)) continue + // Never supplied is not overwritten, and compared by value: a default can be an + // object, where identity would report every correct run as overridden. + if (result[key] !== undefined && !deepEqual(result[key], prop.default)) resetKeys.push(key) + result[key] = prop.default + } + return { args: { ...result }, resetKeys } +} + +/** How a form says what {@link enforceDisabledDefaults} overwrote, shared by the two that + * run it so the wording cannot drift apart. */ +export const resetKeysToast = (resetKeys: string[]): string => + `Disabled field${resetKeys.length > 1 ? 's' : ''} ${resetKeys + .map((k) => `'${k}'`) + .join(', ')} reset to default value${resetKeys.length > 1 ? 's' : ''}` + +/** Types `setInputCat` routes to a widget bound to a scalar. */ +const SCALAR_TYPES = new Set(['string', 'number', 'integer', 'boolean']) + +/** + * Declares an array though its `type` says `object`. A mismatch here throws rather than + * reading wrong: `MultiSelect` maps over the value as it renders, so anything else takes + * the form down, Cancel with it — a reference included, since it draws before resolving. + */ +const declaresDynMultiselect = (prop: any) => + typeof prop?.format === 'string' && prop.format.startsWith('dynmultiselect-') + +const fitsScalarType = (value: any, type: string): boolean => + type === 'integer' ? typeof value === 'number' : typeof value === type + +/** + * Resolved at run time, so the declared type describes what the job receives and never the + * string standing in for it. `ArgInput.validateInput` blesses these ahead of every type check. + */ +const REFERENCE_PREFIXES = ['$var:', '$res:', '$jsonvar:'] +const isReference = (value: any): boolean => + typeof value === 'string' && REFERENCE_PREFIXES.some((prefix) => value.startsWith(prefix)) + +/** No plain reading in the declared type; distinct from a value that reads as `undefined`. */ +const UNCOERCIBLE = Symbol('uncoercible') + +/** + * The value a scalar widget would stand for, or {@link UNCOERCIBLE}. Only conversions with + * one plain reading: a number input shows `"7"` as 7 and a toggle shows any non-empty + * string as on, so guessing past this would put a value on screen that nobody wrote. + */ +function coerceScalar(value: any, type: string): any { + if (typeof value === 'object') return UNCOERCIBLE + if (type === 'string') { + return typeof value === 'number' || typeof value === 'boolean' ? String(value) : UNCOERCIBLE + } + if (typeof value !== 'string') return UNCOERCIBLE + const trimmed = value.trim() + if (type === 'number' || type === 'integer') { + if (trimmed === '') return UNCOERCIBLE + const parsed = Number(trimmed) + return Number.isFinite(parsed) ? parsed : UNCOERCIBLE + } + if (type === 'boolean') { + if (trimmed.toLowerCase() === 'true') return true + if (trimmed.toLowerCase() === 'false') return false + } + return UNCOERCIBLE +} + +/** + * Drop every argument the schema does not declare, naming them. The schema is what every run + * surface builds its fields from, so a value under a name it never declares has no widget + * anywhere: sending one is sending what nobody could see or edit before the run. + */ +export function dropUndeclaredArgs( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; undeclaredKeys: string[] } { + const properties = schema?.properties ?? {} + // hasOwn, not `in`: every object inherits `constructor` and `toString`, so `in` would + // hand an inherited declaration to an argument the schema never named. + const kept: Record = Object.create(null) + const undeclaredKeys: string[] = [] + for (const [key, value] of Object.entries(args ?? {})) { + if (Object.hasOwn(properties, key)) kept[key] = value + else undeclaredKeys.push(key) + } + return { args: { ...kept }, undeclaredKeys } +} + +/** + * Make arguments say what the run form will show, then apply {@link enforceDisabledDefaults}. + * A scalar widget renders its own reading of a wrong-typed value and never writes it back, so + * an untouched form would submit what it never displayed; a value with no reading is cleared. + * Top-level only: descending means resolving `oneOf`, where being wrong rewrites user input. + */ +export function coerceArgsToSchema( + args: Record, + schema: { properties?: Record } | undefined +): { + args: Record + resetKeys: string[] + clearedKeys: string[] + undeclaredKeys: string[] +} { + const properties = schema?.properties ?? {} + const clearedKeys: string[] = [] + const { args: declared, undeclaredKeys } = dropUndeclaredArgs(args, schema) + const kept: Record = Object.create(null) + for (const [key, value] of Object.entries(declared)) { + // Declared, but a declaration can still be nothing, and reading `.type` off it throws. + const prop = properties[key] + if ( + prop === undefined || + value == null || + (isReference(value) && !declaresDynMultiselect(prop)) + ) { + kept[key] = value + continue + } + if (declaresDynMultiselect(prop)) { + if (Array.isArray(value)) kept[key] = value + else clearedKeys.push(key) + continue + } + if (!SCALAR_TYPES.has(prop.type) || fitsScalarType(value, prop.type)) { + kept[key] = value + continue + } + const coerced = coerceScalar(value, prop.type) + if (coerced === UNCOERCIBLE) clearedKeys.push(key) + else kept[key] = coerced + } + const { args: result, resetKeys } = enforceDisabledDefaults({ ...kept }, schema) + return { args: result, resetKeys, clearedKeys, undeclaredKeys } +} + +/** + * Every bag of `properties` a declaration can show a value's keys through, including every + * `oneOf` branch rather than the selected one: a secret under a variant nobody opened + * leaves the form just the same. + */ +function declarationBags(prop: any): Record[] { + const bags: Record[] = [] + if (prop?.properties) bags.push(prop.properties) + if (Array.isArray(prop?.oneOf)) + for (const branch of prop.oneOf) if (branch?.properties) bags.push(branch.properties) + return bags +} + +/** + * Apply `visit` to every value whose declaration matches `isLeaf`, at any depth; returning + * `undefined` removes it. Recursive because the form is, so a level left unvisited is one a + * secret can sit at. Descends on the value's shape, never on the declaration's keys: one + * carrying both `items` and `properties` must not route a shape down the other's branch. + */ +function mapLeaves( + value: any, + prop: any, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown, + path: (string | number)[] +): any { + if (value == null || typeof value !== 'object') return value + // A container shaped unlike its declaration is kept rather than dropped, since the widget + // is the one that reports it — so the walk has to reach in through whichever half the + // declaration does carry, or a secret under one leaves the form verbatim. + if (Array.isArray(value)) + return value.map((item, i) => mapLeaves(item, prop?.items ?? prop, isLeaf, visit, [...path, i])) + const bags = declarationBags(prop) + if (bags.length === 0) + return prop?.items ? mapLeaves(value, prop.items, isLeaf, visit, path) : value + // Null prototype, and keyed off the value rather than the declaration: a key is only + // ever rewritten where it already exists, so no branch of a `oneOf` can add one. + const result: Record = Object.assign(Object.create(null), value) + for (const key of Object.keys(result)) { + const declared = bags.filter((bag) => Object.hasOwn(bag, key)).map((bag) => bag[key]) + // Segments, never a joined name: a key can itself hold a dot, and two leaves reported + // under one name let a caller correlating by it take the one for the other. + const keyPath = [...path, key] + // A matching object is a leaf, not a level: a password object is stored whole as a + // single $jsonvar: reference, and a file is one opaque base64 string. + const leaf = declared.find(isLeaf) + if (leaf) { + const mapped = visit(result[key], leaf, keyPath) + if (mapped === undefined) delete result[key] + else result[key] = mapped + continue + } + for (const declaration of declared) + result[key] = mapLeaves(result[key], declaration, isLeaf, visit, keyPath) + } + return { ...result } +} + +export const isSecretProp = (prop: any) => !!prop?.password + +const isFileProp = (prop: any) => + prop?.contentEncoding === 'base64' || prop?.items?.contentEncoding === 'base64' + +function fileMarker(base64: string): string { + const bytes = Math.floor((base64.length * 3) / 4) + return bytes < 1024 * 1024 + ? `` + : `` +} + +/** {@link mapLeaves} over a whole argument object, against a schema that may declare + * nothing to match. Copies either way, for the reason {@link enforceDisabledDefaults} + * copies. */ +export function mapArgLeaves( + args: Record, + schema: { properties?: Record } | undefined, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown +): Record { + return mapLeaves(args ?? {}, { properties: schema?.properties ?? {} }, isLeaf, visit, []) +} + +/** A leaf's path as the lines naming it to a reader read: `creds[0].secret`. */ +const formatArgPath = (path: (string | number)[]): string => + path.reduce( + (acc, segment) => + typeof segment === 'number' + ? `${acc}[${segment}]` + : acc + ? `${acc}.${segment}` + : String(segment), + '' + ) + +/** + * Drop every file argument, so a caller cannot propose file bytes on the user's behalf: + * the field opens empty and the user attaches the file. Bytes a form is prefilled with + * are bytes the stored transcript carries, unbounded, for a value no caller can produce. + * Reports the path of each one removed, or the caller reads the absence as the user having + * deleted the value. + */ +export function stripFileArgs( + args: Record, + schema: { properties?: Record } | undefined, + strippedKeys?: string[] +): Record { + return mapArgLeaves(args, schema, isFileProp, (value, _prop, path) => { + if (value !== undefined) strippedKeys?.push(formatArgPath(path)) + return undefined + }) +} + +/** + * Replace a sensitive value with a fixed marker, for text that leaves the form. A reference is + * kept: it names a variable rather than holding one, and the run page shows the same job's + * arguments that way. An empty field is kept for the reason `processSecretArgs` mints nothing + * for one — marking it would describe a secret the run never carried. + */ +export function redactSecretArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + return mapArgLeaves(args, schema, isSecretProp, (value) => + value == null ? undefined : value === '' || isReference(value) ? value : '' + ) +} + +/** + * Replace every file argument with a marker naming its size. The base64 belongs in the + * job request and nowhere else: rendered it is unreadable, persisted it is unbounded, and + * a file small enough to survive truncation reaches the model whole. + */ +export function redactFileArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + const mark = (value: unknown) => (typeof value === 'string' ? fileMarker(value) : value) + return mapArgLeaves(args, schema, isFileProp, (value) => + Array.isArray(value) ? value.map(mark) : mark(value) + ) +} + export function isWindmillTooBigObject(obj: any): boolean { return ( typeof obj === 'object' && diff --git a/frontend/src/lib/components/markdownProse.ts b/frontend/src/lib/components/markdownProse.ts index aedd2f45b1..3f79ab79a5 100644 --- a/frontend/src/lib/components/markdownProse.ts +++ b/frontend/src/lib/components/markdownProse.ts @@ -4,10 +4,14 @@ * call site with layout-only classes (padding, width, bg); anything typographic * belongs here. * - * - 'xs': micro scale for dense secondary panes (chat reasoning blocks) + * - 'xs': micro scale for dense secondary panes (chat reasoning blocks, group notes) * - 'sm': compact chat-bubble scale (assistant messages, flow/app chat, settings) * - 'doc': same rhythm and body size as 'sm', with a taller heading ramp * (lg/base/sm) and semibold headings for document-like surfaces (artifacts) + * + * h1 and h2 step above the body size in every preset, so `#` and `##` read as + * headings rather than bold body text. Below that the tight 'xs' and 'sm' scales + * run out of room, and h3 down differentiates by weight and colour alone. */ // Kept as literal template parts: Tailwind's scanner reads class names verbatim @@ -28,8 +32,8 @@ const bodyXs = 'text-primary prose-p:text-primary prose-li:text-primary prose-p:text-xs prose-li:text-xs prose-code:text-xs prose-pre:text-xs prose-table:text-xs' export const markdownProse = { - xs: `${base} prose-sm leading-snug prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-2xs prose-h2:text-2xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, - sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-sm prose-h2:text-xs prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, + xs: `${base} prose-sm leading-snug prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-sm prose-h2:text-xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, + sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-base prose-h2:text-sm prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, doc: `${base} ${rhythm} ${bodyXs} prose-headings:mt-8 prose-headings:mb-2 prose-headings:font-semibold prose-headings:text-emphasis prose-h1:text-lg prose-h2:text-base prose-h3:text-sm prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs prose-pre:bg-transparent prose-pre:p-0` } as const diff --git a/frontend/src/lib/components/mcp/McpScopeSelector.svelte b/frontend/src/lib/components/mcp/McpScopeSelector.svelte index a240b09db7..46274d10d0 100644 --- a/frontend/src/lib/components/mcp/McpScopeSelector.svelte +++ b/frontend/src/lib/components/mcp/McpScopeSelector.svelte @@ -5,8 +5,10 @@ import Popover from '$lib/components/Popover.svelte' import MultiSelect from '$lib/components/select/MultiSelect.svelte' import { safeSelectItems } from '$lib/components/select/utils.svelte' + import { integrationDisplayName } from '$lib/components/resourceTypeDisplay' + import { listHubIntegrationsShared } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { FlowService, FolderService, IntegrationService, ScriptService } from '$lib/gen' + import { FlowService, FolderService, ScriptService } from '$lib/gen' import { mcpEndpointTools } from '$lib/mcpEndpointTools' import { endpointPathPolicy, @@ -268,11 +270,8 @@ if (allApps.length > 0) return try { loadingApps = true - allApps = ( - await IntegrationService.listHubIntegrations({ - kind: 'script' - }) - ).map((x) => x.name) + const integrations = await listHubIntegrationsShared('script') + allApps = integrations.map((x) => x.name) } catch (err) { console.error('Hub is not available') allApps = [] @@ -602,7 +601,9 @@
Error fetching apps
{:else} ({ value: app, label: integrationDisplayName(app) })) + )} placeholder="Select apps" bind:value={newMcpApps} /> diff --git a/frontend/src/lib/components/mcp/McpServerIcon.svelte b/frontend/src/lib/components/mcp/McpServerIcon.svelte new file mode 100644 index 0000000000..b3d005ad7f --- /dev/null +++ b/frontend/src/lib/components/mcp/McpServerIcon.svelte @@ -0,0 +1,29 @@ + + + + diff --git a/frontend/src/lib/components/mcp/enabledServers.ts b/frontend/src/lib/components/mcp/enabledServers.ts index b49b788c67..6dbeb997c1 100644 --- a/frontend/src/lib/components/mcp/enabledServers.ts +++ b/frontend/src/lib/components/mcp/enabledServers.ts @@ -1,11 +1,12 @@ -import { createEnabledPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' +import { createPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' /** Which MCP servers the chat may act through, per workspace and per account. A * server's tools both reach an external system and put their descriptions in the * model's context, so one is off until it is turned on; connecting one through the * chat turns it on for the person who connected it. */ -const preference = createEnabledPathsPreference('wm_mcp_enabled') +const preference = createPathsPreference('wm_mcp_enabled', false) -export const enabledMcpPaths = preference.enabledPaths +/** Servers are off by default, so what is stored as on is the whole enabled set. */ +export const enabledMcpPaths = preference.explicitlyEnabledPaths export const isMcpEnabled = preference.isEnabled export const setMcpEnabled = preference.setEnabled diff --git a/frontend/src/lib/components/mcp/iconCache.ts b/frontend/src/lib/components/mcp/iconCache.ts index e3c0f592e2..59ba63adfb 100644 --- a/frontend/src/lib/components/mcp/iconCache.ts +++ b/frontend/src/lib/components/mcp/iconCache.ts @@ -33,6 +33,18 @@ export function cachedProviderKey( return entry.editedAt === editedAt ? entry.key : undefined } +/** + * The stored mark for a path, ignoring `editedAt`: a transcript row has none to + * match against, and nothing acts on a mark. + */ +export function cachedProviderMark( + workspace: string, + path: string +): { key: string | null } | undefined { + const entry = read()[workspace]?.[path] + return entry ? { key: entry.key } : undefined +} + export function rememberProviderKey( workspace: string, path: string, diff --git a/frontend/src/lib/components/mcp/mcpMenu.svelte.ts b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts index 847c03e61a..a911d71c07 100644 --- a/frontend/src/lib/components/mcp/mcpMenu.svelte.ts +++ b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts @@ -1,4 +1,4 @@ -import { List, Plug, Plus } from 'lucide-svelte' +import { List, Plus } from 'lucide-svelte' import type { Component } from 'svelte' import { get } from 'svelte/store' import { ResourceService } from '$lib/gen' @@ -8,7 +8,9 @@ import type { Item } from '$lib/utils' import type { AIChatManager } from '../copilot/chat/AIChatManager.svelte' import { isMcpEnabled, setMcpEnabled } from './enabledServers' import { cachedProviderKey, rememberProviderKey } from './iconCache' +import { isOwnOrSharedMcpPath, MCP_LIST_PER_PAGE, mcpViewer } from './ownServers' import { loadProviderIcon } from './providerIcon' +import McpServerIcon from './McpServerIcon.svelte' type Row = { path: string @@ -55,17 +57,22 @@ export class McpMenu { async #load(ws: string) { const seq = ++this.#seq try { - const resources = await ResourceService.listResource({ - workspace: ws, - resourceType: 'mcp', - perPage: 100 - }) + const [resources, viewer] = await Promise.all([ + ResourceService.listResource({ + workspace: ws, + resourceType: 'mcp', + perPage: MCP_LIST_PER_PAGE + }), + mcpViewer(ws) + ]) if (seq !== this.#seq) return - this.#rows = resources.map((r) => ({ - path: r.path, - editedAt: r.edited_at, - enabled: isMcpEnabled(ws, r.path) - })) + this.#rows = resources + .filter((r) => isOwnOrSharedMcpPath(r.path, r.extra_perms, viewer)) + .map((r) => ({ + path: r.path, + editedAt: r.edited_at, + enabled: isMcpEnabled(ws, r.path) + })) this.#rowsWorkspace = ws void this.#loadIcons(ws, seq) } catch { @@ -164,19 +171,13 @@ export class McpMenu { return [ ...shown.map(({ path }) => ({ displayName: path, + icon: McpServerIcon, // Getters, not snapshots: the menu stays open across a click, and it has // to read through the live list rather than the row captured here, since // a reload replaces every row object and a getter bound to the old one // would go on reporting the state it was built with. - get icon() { - // Plug where the provider is unknown, so one nameless server does not - // pull its label out of line with the rest. - return row(path)?.icon ?? Plug - }, - // Provider icons take css lengths and ignore lucide's `size`, so without - // this one of them renders at its 24px default among 14px menu icons. get iconProps() { - return row(path)?.icon ? { width: '14px', height: '14px' } : undefined + return { icon: row(path)?.icon, size: 14 } }, get toggle() { return row(path)?.enabled ?? false diff --git a/frontend/src/lib/components/mcp/ownServers.test.ts b/frontend/src/lib/components/mcp/ownServers.test.ts new file mode 100644 index 0000000000..d6982790ba --- /dev/null +++ b/frontend/src/lib/components/mcp/ownServers.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { session, roleMock } = vi.hoisted(() => ({ + session: { workspace_id: 'browsed', username: 'alice', pgroups: ['g/browsed-only'] }, + roleMock: vi.fn() +})) + +vi.mock('$lib/stores', () => ({ + userStore: { subscribe: (run: (v: unknown) => void) => (run({ ...session }), () => {}) } +})) +vi.mock('$lib/user', () => ({ getWorkspaceRole: roleMock })) + +import { isOwnOrSharedMcpPath, mcpViewer } from './ownServers' + +const alice = { username: 'alice', pgroups: ['g/eng'] } + +describe('isOwnOrSharedMcpPath', () => { + it('keeps own, folder, and explicitly shared servers; drops the rest of u/', () => { + expect(isOwnOrSharedMcpPath('u/alice/notion', {}, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('f/team/notion', {}, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('u/bob/notion', {}, alice)).toBe(false) + expect(isOwnOrSharedMcpPath('u/alicex/notion', {}, alice)).toBe(false) + expect(isOwnOrSharedMcpPath('u/bob/notion', { 'u/alice': false }, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('u/bob/notion', { 'g/eng': false }, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('u/bob/notion', { 'g/ops': true }, alice)).toBe(false) + expect(isOwnOrSharedMcpPath('u/alice/notion', {}, { username: undefined, pgroups: [] })).toBe( + false + ) + }) +}) + +describe('mcpViewer', () => { + beforeEach(() => roleMock.mockReset()) + + it('answers from the store only for the browsed workspace', async () => { + expect(await mcpViewer('browsed')).toEqual({ username: 'alice', pgroups: ['g/browsed-only'] }) + expect(roleMock).not.toHaveBeenCalled() + }) + + it('looks the identity up for another workspace instead of reusing the browsed one', async () => { + roleMock.mockResolvedValue({ + kind: 'resolved', + user: { username: 'alice_other', pgroups: ['g/eng'] } + }) + expect(await mcpViewer('other')).toEqual({ username: 'alice_other', pgroups: ['g/eng'] }) + expect(roleMock).toHaveBeenCalledWith('other') + }) + + it('yields no identity when the lookup fails', async () => { + roleMock.mockResolvedValue({ kind: 'lookup_failed' }) + expect(await mcpViewer('other')).toEqual({ username: undefined, pgroups: [] }) + }) +}) diff --git a/frontend/src/lib/components/mcp/ownServers.ts b/frontend/src/lib/components/mcp/ownServers.ts new file mode 100644 index 0000000000..31c860a039 --- /dev/null +++ b/frontend/src/lib/components/mcp/ownServers.ts @@ -0,0 +1,48 @@ +import { get } from 'svelte/store' +import { userStore } from '$lib/stores' +import { getWorkspaceRole } from '$lib/user' + +/** The principals an `extra_perms` entry can name to grant this user access. */ +export type McpViewer = { username: string | undefined; pgroups: string[] } + +/** Wide enough that a workspace's whole MCP catalog arrives at once: the filter + * below runs after the server's LIMIT, so a short page could drop the viewer's + * own rows while foreign `u/` rows fill it. */ +export const MCP_LIST_PER_PAGE = 1000 + +/** + * Whether the chat lists this MCP server. A server under another user's `u/` + * prefix carries that person's credentials, and an admin's database role lets + * the resource listing return it. Acting through it would call the provider + * as them, so the chat only offers it when its owner shared it explicitly. + */ +export function isOwnOrSharedMcpPath( + path: string, + extraPerms: Record | undefined, + viewer: McpViewer +): boolean { + if (!path.startsWith('u/')) return true + if (viewer.username !== undefined && path.startsWith(`u/${viewer.username}/`)) return true + if (!extraPerms) return false + return ( + (viewer.username !== undefined && `u/${viewer.username}` in extraPerms) || + viewer.pgroups.some((g) => g in extraPerms) + ) +} + +/** + * The viewer's identity in `workspace`. Username and groups are per workspace, + * and a session chat can operate on a workspace other than the one being + * browsed, so `userStore` only answers when it describes that same workspace. + * A failed lookup yields no username: every `u/` server is then hidden rather + * than judged against another workspace's identity. + */ +export async function mcpViewer(workspace: string): Promise { + const u = get(userStore) + if (u?.workspace_id === workspace) return { username: u.username, pgroups: u.pgroups ?? [] } + const role = await getWorkspaceRole(workspace) + if (role.kind === 'resolved') { + return { username: role.user.username, pgroups: role.user.pgroups ?? [] } + } + return { username: undefined, pgroups: [] } +} diff --git a/frontend/src/lib/components/mcp/serverMark.test.ts b/frontend/src/lib/components/mcp/serverMark.test.ts new file mode 100644 index 0000000000..7e34d27284 --- /dev/null +++ b/frontend/src/lib/components/mcp/serverMark.test.ts @@ -0,0 +1,45 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { getResourceMock } = vi.hoisted(() => ({ getResourceMock: vi.fn() })) + +vi.mock('$lib/gen', () => ({ ResourceService: { getResource: getResourceMock } })) +vi.mock('./iconCache', () => ({ cachedProviderMark: () => undefined })) +vi.mock('./providerIcon', () => ({ + loadProviderIcon: async (key: string | null) => (key ? `icon:${key}` : undefined), + providerKey: (url: unknown) => + typeof url === 'string' && url.includes('linear') ? 'linear' : null +})) + +import { forgetMcpServerMarks, resolveMcpServerMark } from './serverMark' + +describe('resolveMcpServerMark', () => { + beforeEach(() => { + forgetMcpServerMarks() + getResourceMock.mockReset() + }) + + it('reads a server once and shares the answer', async () => { + getResourceMock.mockResolvedValue({ value: { url: 'https://mcp.linear.app/mcp' } }) + + const marks = await Promise.all([ + resolveMcpServerMark('ws', 'u/admin/linear_mcp'), + resolveMcpServerMark('ws', 'u/admin/linear_mcp') + ]) + + expect(marks.map((m) => m.icon)).toEqual(['icon:linear', 'icon:linear']) + expect(getResourceMock).toHaveBeenCalledTimes(1) + }) + + // A failure must not be memoized: one offline blip would otherwise leave the + // server unmarked in every later row until the page reloads. + it('retries after a failed read', async () => { + getResourceMock.mockRejectedValueOnce(new Error('offline')) + expect(await resolveMcpServerMark('ws', 'u/admin/linear_mcp')).toEqual({}) + + getResourceMock.mockResolvedValue({ value: { url: 'https://mcp.linear.app/mcp' } }) + const mark = await resolveMcpServerMark('ws', 'u/admin/linear_mcp') + + expect(mark.icon).toBe('icon:linear') + expect(getResourceMock).toHaveBeenCalledTimes(2) + }) +}) diff --git a/frontend/src/lib/components/mcp/serverMark.ts b/frontend/src/lib/components/mcp/serverMark.ts new file mode 100644 index 0000000000..c728abdead --- /dev/null +++ b/frontend/src/lib/components/mcp/serverMark.ts @@ -0,0 +1,47 @@ +import type { Component } from 'svelte' +import { ResourceService } from '$lib/gen' +import { cachedProviderMark } from './iconCache' +import { loadProviderIcon, providerKey } from './providerIcon' + +/** Windmill's own icon for a connected server's integration, when it ships one. */ +export type McpServerMark = { icon?: Component } + +// One resolution per server per session, shared by every transcript row naming it — +// a chat can hold dozens of calls against the same server. +const marks = new Map>() + +export function resolveMcpServerMark(workspace: string, path: string): Promise { + const key = `${workspace}:${path}` + let pending = marks.get(key) + if (!pending) { + // A failed read is dropped rather than memoized: one offline blip or a 401 during + // a token refresh would otherwise leave the server unmarked in every later row + // until the page reloads. + pending = load(workspace, path).catch(() => { + marks.delete(key) + return {} + }) + marks.set(key, pending) + } + return pending +} + +/** + * Forget what was resolved, for the settings section to call when it reloads the + * connections: a path can be reconnected to a different provider, and a mark held for + * the life of the page would go on marking new call rows with the old provider's icon. + */ +export function forgetMcpServerMarks() { + marks.clear() +} + +async function load(workspace: string, path: string): Promise { + const cached = cachedProviderMark(workspace, path) + if (cached) return { icon: await loadProviderIcon(cached.key) } + // Deliberately not written back to the shared cache: that entry is keyed by + // `editedAt` for the server list's sake, and storing one from here — where the + // row is a past call and `editedAt` is unknown — would make every list re-read. + const resource = await ResourceService.getResource({ workspace, path }) + const url = (resource.value as { url?: unknown } | undefined)?.url + return { icon: await loadProviderIcon(providerKey(url)) } +} diff --git a/frontend/src/lib/components/meltComponents/Popover.svelte b/frontend/src/lib/components/meltComponents/Popover.svelte index f850e53f6c..6a6b7d681b 100644 --- a/frontend/src/lib/components/meltComponents/Popover.svelte +++ b/frontend/src/lib/components/meltComponents/Popover.svelte @@ -60,6 +60,8 @@ documentationLink?: string | undefined disableFocusTrap?: boolean openFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined + /** Element to focus when the popover closes; defaults to the trigger, `null` leaves focus alone. */ + closeFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined escapeBehavior?: EscapeBehaviorType enableFlyTransition?: boolean onKeyDown?: (e: KeyboardEvent) => void @@ -99,6 +101,7 @@ documentationLink = undefined, disableFocusTrap = false, openFocus = undefined, + closeFocus = undefined, escapeBehavior = 'close', enableFlyTransition = false, onKeyDown = () => {}, @@ -133,6 +136,7 @@ disableFocusTrap: untrack(() => disableFocusTrap), escapeBehavior: untrack(() => escapeBehavior), openFocus: untrack(() => openFocus), + closeFocus: untrack(() => closeFocus), onOpenChange: ({ curr, next }) => { if (curr != next) { dispatch('openChange', next) diff --git a/frontend/src/lib/components/pickerPopularity.test.ts b/frontend/src/lib/components/pickerPopularity.test.ts new file mode 100644 index 0000000000..f9440981ad --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from 'vitest' +import { alphabetical, byPopularity, totalLocalCountsByApp } from './pickerPopularity' + +const order = (names: string[], hub: Record, local: Record = {}) => + [...names].sort(byPopularity(hub, local)) + +describe('byPopularity', () => { + // The tier that stops a filling-up hub from squeezing the workspace's own stack out of + // the ordering: a global pick count grows without bound, a local one does not. + it('leads with what the workspace uses, whatever the hub says', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { stripe: 1 })).toEqual(['stripe', 'slack']) + }) + + it('ranks the used types among themselves by hub picks', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { slack: 1, stripe: 1 })).toEqual([ + 'slack', + 'stripe' + ]) + }) + + it('breaks a hub tie on how much the workspace uses it', () => { + expect(order(['slack', 'stripe'], { slack: 5, stripe: 5 }, { slack: 1, stripe: 2 })).toEqual([ + 'stripe', + 'slack' + ]) + }) + + it('ranks the unused types by hub picks, below every used one', () => { + expect(order(['ably', 'github', 'stripe'], { ably: 900, github: 5 }, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + it('falls back to alphabetical for everything neither signal ranks', () => { + expect(order(['stripe', 'ably', 'github'], { github: 3 })).toEqual(['github', 'ably', 'stripe']) + }) + + it('orders on local usage alone when the hub ranks nothing', () => { + expect(order(['stripe', 'ably', 'github'], {}, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + // The lists render before either signal lands, and one of them arrives in a server-side + // HashMap's iteration order, so the resting comparator has to sort rather than no-op. + it('leaves an alphabetical order with no signal at all', () => { + expect(['stripe', 'ably', 'github'].sort(alphabetical)).toEqual(['ably', 'github', 'stripe']) + }) +}) + +describe('totalLocalCountsByApp', () => { + const HUB = [ + { name: 'discord_webhook', app: 'discord', picks: 0 }, + { name: 'discord_bot_configuration', app: 'discord', picks: 0 }, + { name: 'ms_teams_webhook', app: 'msteams', picks: 0 }, + { name: 'slack', app: 'slack', picks: 0 } + ] + + // The integration pickers list app names, the counts arrive keyed by resource type, and + // the two only usually agree. Without the mapping a workspace whose Discord credential is + // a `discord_webhook` never reaches the used-here tier at all. + it('totals a resource type under the integration it belongs to', () => { + expect(totalLocalCountsByApp({ discord_webhook: 2 }, HUB)).toEqual({ discord: 2 }) + }) + + it('sums the several types one integration can have', () => { + expect( + totalLocalCountsByApp({ discord_webhook: 2, discord_bot_configuration: 1 }, HUB) + ).toEqual({ discord: 3 }) + }) + + // What a workspace-made type, and an unreachable hub, both leave every entry with. + it('keeps a type the hub has no mapping for under its own name', () => { + expect(totalLocalCountsByApp({ c_acme: 1, slack: 2 }, HUB)).toEqual({ c_acme: 1, slack: 2 }) + }) +}) diff --git a/frontend/src/lib/components/pickerPopularity.ts b/frontend/src/lib/components/pickerPopularity.ts new file mode 100644 index 0000000000..2d585ff858 --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.ts @@ -0,0 +1,149 @@ +import { get } from 'svelte/store' +import { ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { isCustomResourceTypeName } from './resourceTypeDisplay' + +/** + * How often something has been picked or used, keyed by integration or resource type name. + * A name the caller lists but this map does not mention counts as zero, which is what an + * unpicked entry and an absent signal both mean. + */ +export type PopularityCounts = Record + +/** + * The signals are read on every picker open, so they are cached briefly; both resolve to an + * empty map rather than rejecting, since an ordering hint is never worth a broken list. + */ +const CACHE_MS = 60_000 + +type HubResourceTypeInfo = { name: string; app: string; picks: number } + +const hubInfoCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + return await ResourceService.listHubResourceTypeInfo({ workspace }) + } catch { + return [] + } + }, + { invalidateMs: CACHE_MS } +) + +const localCountsCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + const counts = await ResourceService.listResourceCountsByType({ workspace }) + return Object.fromEntries(counts.map((c) => [c.resource_type, c.count])) + } catch { + return {} + } + }, + { invalidateMs: CACHE_MS } +) + +/** + * What the hub sees people pick, per resource type. Empty on a hub that counts nothing, + * and on an instance that has switched the hub off — a closed environment must not spend a + * request on hub.windmill.dev just to order a list. + */ +export async function hubResourceTypePicks(workspace: string): Promise { + if (get(disableHubStore)) return {} + const info = await hubInfoCached({ workspace }) + return Object.fromEntries(info.map((rt) => [rt.name, rt.picks])) +} + +/** + * How many resources of each type this workspace holds — the only evidence about this + * particular team. Keyed by resource type, which is what the add-resource drawer lists. + */ +export function localResourceTypeCounts(workspace: string): Promise { + return localCountsCached({ workspace }) +} + +/** + * The same counts totalled per integration, which is what the flow step picker lists. + * + * A type usually shares its integration's name, but often enough it does not: + * `discord_webhook` and `discord_bot_configuration` are both Discord, `ms_teams_webhook` and + * `azure_bot` are both MS Teams. Only the hub knows that, so a workspace whose Discord + * credential is a `discord_webhook` would otherwise read as one that has never touched + * Discord — and since local usage is the leading tier, that decides which half of the list + * the integration lands in, not merely its position within one. + * + * A type the hub has no mapping for counts under its own name, which is the right guess and + * also what an unreachable hub leaves every type with. + */ +export async function localCountsByIntegration(workspace: string): Promise { + const [counts, info] = await Promise.all([ + localCountsCached({ workspace }), + get(disableHubStore) ? Promise.resolve([]) : hubInfoCached({ workspace }) + ]) + return totalLocalCountsByApp(counts, info) +} + +/** The mapping half of {@link localCountsByIntegration}, separated so it can be tested alone. */ +export function totalLocalCountsByApp( + counts: PopularityCounts, + hub: { name: string; app: string }[] +): PopularityCounts { + const appOf = new Map(hub.map((rt) => [rt.name, rt.app])) + const byApp: PopularityCounts = {} + for (const [name, count] of Object.entries(counts)) { + const app = appOf.get(name) ?? name + byApp[app] = (byApp[app] ?? 0) + count + } + return byApp +} + +/** + * Tell the hub a resource type was taken into a workspace, which is what its ranking counts. + * Fire-and-forget: a hub that does not count picks must not be felt by the user who just + * saved a resource. Workspace-made types exist on no hub, so they are not reported. + */ +export function recordHubResourceTypePick(workspace: string, resourceType: string): void { + if (get(disableHubStore)) return + if (!resourceType || isCustomResourceTypeName(resourceType)) return + ResourceService.pickHubResourceType({ workspace, name: resourceType }).catch(() => {}) +} + +/** + * Orders the lists that offer hub content: integrations in the flow step picker, resource + * types in the add-resource drawer. + * + * Four tiers. **Whether this workspace already holds a resource of the type leads**, then the + * hub's pick count, then how many local resources there are, then the name. + * + * Used-here leads rather than merely breaking hub ties because the two counts are on + * incomparable scales: a hub pick count is global and grows without bound, a local count is + * usually single digits. Ranked the other way round, local usage only ever sorts the slice + * where hub counts are equal — which, since they are distinct integers, is just the tail + * that nobody has picked. That reads fine on a hub with few picks and silently stops + * mattering as one fills up, so the ordering would drift away from the workspace's own + * stack with no change to this code. + * + * Within each half the hub decides, so "yours" and "everyone's" are both honoured rather + * than blended with a weighting constant that would need tuning. Alphabetical is the floor, + * and it is where an entry neither signal knows about lands. + */ +export function byPopularity( + hub: PopularityCounts, + local: PopularityCounts +): (a: string, b: string) => number { + const usedHere = (name: string) => ((local[name] ?? 0) > 0 ? 1 : 0) + return (a, b) => + usedHere(b) - usedHere(a) || + (hub[b] ?? 0) - (hub[a] ?? 0) || + (local[b] ?? 0) - (local[a] ?? 0) || + a.localeCompare(b) +} + +/** + * The ordering to hold before either signal has landed: the alphabetical floor, which is + * what `byPopularity` degrades to anyway. + * + * A list has to be sorted by *something* from its first paint — one source of these names + * is a `HashMap` on the server, so leaving them unsorted means hash order, which differs + * between processes. + */ +export const alphabetical = byPopularity({}, {}) diff --git a/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts b/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts index 444e03a0a3..6a18a28e46 100644 --- a/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts +++ b/frontend/src/lib/components/raw_apps/rawAppDiffUtils.ts @@ -1,5 +1,6 @@ import { extToLang } from '$lib/editorLangUtils' -import { cleanValueProperties, orderedYamlStringify, replaceFalseWithUndefined } from '$lib/utils' +import { cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' +import { orderedYamlStringify } from '$lib/utils/orderedYaml' // A raw app rendered as a *folder of files* for diffing. Each entry is one // virtual file: real `files` keep their natural path, runnables become diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.ts b/frontend/src/lib/components/raw_apps/sdkScopes.ts index f0ece00c3e..b0177bb660 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.ts @@ -28,6 +28,16 @@ export const FRONTEND_SDK_SCOPES: { value: string; label: string; description: s value: 'variables:read', label: 'Read variables', description: 'Read variable values the viewer can access' + }, + { + value: 'flow_conversations:read', + label: 'Read your flow chats', + description: 'List your chat conversations with flows and read their messages' + }, + { + value: 'flow_conversations:write', + label: 'Manage your flow chats', + description: 'Read and delete your chat conversations with flows' } ] @@ -72,5 +82,5 @@ export function storeSdkConsent( ): void { try { localStorage.setItem(sdkConsentKey(viewer, workspace, path), JSON.stringify(scopes)) - } catch (_) { } + } catch (_) {} } diff --git a/frontend/src/lib/components/resourceTypeDisplay.ts b/frontend/src/lib/components/resourceTypeDisplay.ts index 748a2147f9..ad01c05bb9 100644 --- a/frontend/src/lib/components/resourceTypeDisplay.ts +++ b/frontend/src/lib/components/resourceTypeDisplay.ts @@ -1,3 +1,5 @@ +import { SvelteMap } from 'svelte/reactivity' + /** * Resource types are named by abbreviation — `gdrive`, `gcal`, `s3` — so a product's real * name ("Google Drive", "Amazon S3") only ever appears in its description. Matching on the @@ -179,22 +181,51 @@ const RESOURCE_TYPE_WORDS: Record = { woocommerce: 'WooCommerce' } -/** Types whose label is not their name with the parts re-cased. */ -const RESOURCE_TYPE_NAMES: Record = { - adobe_acrobat_sign: 'Adobe Acrobat Sign', - bamboo_hr: 'BambooHR', - cacertificate: 'CA certificate', - deep_infra: 'DeepInfra', - gcal: 'Google Calendar', - gdocs: 'Google Docs', - gdrive: 'Google Drive', - gforms: 'Google Forms', - gsheets: 'Google Sheets', - gworkspace: 'Google Workspace', - ms_sql_server: 'Microsoft SQL Server', - sage_intacct: 'Sage Intacct', - sensortower: 'Sensor Tower', - their_stack: 'TheirStack' +/** + * Names for the items no rule names (`gsheets` -> Google Sheets): stored with resource types, + * and curated by the hub for integrations. Unnamed items are left to the word table above on + * purpose. Reactive because they fill in after first render: a label already on screen updates + * when they land. + */ +const resourceTypeNames = new SvelteMap() +const hubIntegrationNames = new SvelteMap() + +type Named = { name: string; display_name?: string | null } + +/** + * Updates only the names a listing carries: one it carries without a label drops any label kept + * for it, which is how a hub predating display names reverts to inferred labels. Names it does not + * carry are left alone on purpose, since callers pass `kind`-filtered listings. A listing can carry + * one name twice (a workspace's own copy of a hub type beside the `admins` row, in no fixed order), + * so a label from either entry wins over none. + */ +function recordNames(names: SvelteMap, entries: Named[]): void { + const labels = new Map() + for (const entry of entries) { + const label = entry.display_name?.trim() + if (label) labels.set(entry.name, label) + else if (!labels.has(entry.name)) labels.set(entry.name, undefined) + } + for (const [name, label] of labels) { + if (label) names.set(name, label) + else names.delete(name) + } +} + +export function setResourceTypeDisplayNames(types: Named[]): void { + recordNames(resourceTypeNames, types) +} + +/** + * Records the name a single row carries, and never clears one: reading one type can return a + * workspace's nameless copy of a named hub type. A name the hub drops is cleared by the next listing. + */ +export function addResourceTypeDisplayName(type: Named): void { + if (type.display_name?.trim()) recordNames(resourceTypeNames, [type]) +} + +export function setHubIntegrationDisplayNames(integrations: Named[]): void { + recordNames(hubIntegrationNames, integrations) } /** The prefix the resources page puts on a type created in a workspace. */ @@ -205,17 +236,41 @@ export function isCustomResourceTypeName(name: string): boolean { } /** - * Display name for a resource type: `adobe_acrobat_sign` -> `Adobe Acrobat Sign`, `mysql` -> - * `MySQL`, `c_acme_api` -> `Acme API`. Inferred from the name, since nothing in the type - * carries a product name — the two tables above only cover what the inference gets wrong. + * Display name for a resource type: `gsheets` -> `Google Sheets` where the type stores that name, + * otherwise inferred from the type name (`mysql` -> `MySQL`, `c_acme_api` -> `Acme API`), the + * word table above covering what capitalizing gets wrong. */ export function resourceTypeDisplayName(name: string): string { - const exact = RESOURCE_TYPE_NAMES[name] - if (exact) return exact - const stripped = isCustomResourceTypeName(name) ? name.slice(CUSTOM_TYPE_PREFIX.length) : name - return stripped - .split('_') - .map((word) => RESOURCE_TYPE_WORDS[word] ?? word.charAt(0).toUpperCase() + word.slice(1)) + return ( + resourceTypeNames.get(name) ?? + titleize(isCustomResourceTypeName(name) ? name.slice(CUSTOM_TYPE_PREFIX.length) : name) + ) +} + +/** + * Display name for a hub integration slug, as the hub pickers label their filters: + * `activecampaign` -> `ActiveCampaign` from a hub that names it, `Activecampaign` from one + * that does not. + */ +export function integrationDisplayName(app: string): string { + return hubIntegrationNames.get(app) ?? titleize(app) +} + +/** + * Lowercases each word before the lookup and splits on `-` too: a private hub can predate the + * slug rule, and still serve `aws-ses` or `RSS`. `Object.hasOwn`, because a word like + * `constructor` would otherwise resolve up the object literal's prototype chain. + */ +function titleize(name: string): string { + return name + .split(/[_-]/) + .filter(Boolean) + .map((word) => { + const lower = word.toLowerCase() + return Object.hasOwn(RESOURCE_TYPE_WORDS, lower) + ? RESOURCE_TYPE_WORDS[lower] + : word.charAt(0).toUpperCase() + word.slice(1) + }) .join(' ') } diff --git a/frontend/src/lib/components/resourceTypeMatchRank.test.ts b/frontend/src/lib/components/resourceTypeMatchRank.test.ts index 8c97b6bfb1..8b7e2fef60 100644 --- a/frontend/src/lib/components/resourceTypeMatchRank.test.ts +++ b/frontend/src/lib/components/resourceTypeMatchRank.test.ts @@ -1,7 +1,11 @@ import { describe, it, expect } from 'vitest' import { addResourceTitle, + addResourceTypeDisplayName, + integrationDisplayName, resourceTypeDisplayName, + setHubIntegrationDisplayNames, + setResourceTypeDisplayNames, sortResourceTypesByMatch } from './resourceTypeDisplay' @@ -54,7 +58,7 @@ describe('sortResourceTypesByMatch', () => { }) describe('resourceTypeDisplayName', () => { - it('takes the whole-name override when there is one', () => { + it('cases each word from the word table', () => { expect(resourceTypeDisplayName('ms_sql_server')).toBe('Microsoft SQL Server') }) @@ -65,6 +69,46 @@ describe('resourceTypeDisplayName', () => { it('capitalizes anything the tables do not cover', () => { expect(resourceTypeDisplayName('stripe')).toBe('Stripe') }) + + it('prefers the name stored with the type, and infers one where it has none', () => { + expect(resourceTypeDisplayName('gsheets')).toBe('Gsheets') + setResourceTypeDisplayNames([ + { name: 'gsheets', display_name: 'Google Sheets' }, + { name: 'smtp', display_name: null } + ]) + expect(resourceTypeDisplayName('gsheets')).toBe('Google Sheets') + expect(resourceTypeDisplayName('smtp')).toBe('SMTP') + }) + + it('keeps a name when a nameless row shares it, in either order', () => { + for (const rows of [ + [{ name: 'gdrive', display_name: 'Google Drive' }, { name: 'gdrive' }], + [{ name: 'gdrive' }, { name: 'gdrive', display_name: 'Google Drive' }] + ]) { + setResourceTypeDisplayNames([{ name: 'gdrive' }]) + expect(resourceTypeDisplayName('gdrive')).toBe('Gdrive') + setResourceTypeDisplayNames(rows) + expect(resourceTypeDisplayName('gdrive')).toBe('Google Drive') + } + // Reading one type never clears a name: that row can be the nameless copy. + addResourceTypeDisplayName({ name: 'gdrive' }) + expect(resourceTypeDisplayName('gdrive')).toBe('Google Drive') + }) +}) + +describe('integrationDisplayName', () => { + it('reverts to the inferred name once a hub stops sending one', () => { + setHubIntegrationDisplayNames([{ name: 'activecampaign', display_name: 'ActiveCampaign' }]) + expect(integrationDisplayName('activecampaign')).toBe('ActiveCampaign') + // A private hub predating display names omits the field altogether. + setHubIntegrationDisplayNames([{ name: 'activecampaign' }]) + expect(integrationDisplayName('activecampaign')).toBe('Activecampaign') + }) + + it('cases slugs from a hub predating the slug rule', () => { + expect(integrationDisplayName('aws-lambda')).toBe('AWS Lambda') + expect(integrationDisplayName('RSS')).toBe('RSS') + }) }) describe('addResourceTitle', () => { diff --git a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts index 5d2f878bfd..828f060972 100644 --- a/frontend/src/lib/components/runs/useJobsLoader.svelte.ts +++ b/frontend/src/lib/components/runs/useJobsLoader.svelte.ts @@ -18,6 +18,9 @@ import { CancelablePromiseUtils } from '$lib/cancelable-promise-utils' import type { Timeframe } from './timeframes' import { allowWildcards as _allowWildcards, type RunsFilterInstance } from './runsFilter' +// windmill_common::utils::MAX_PER_PAGE: the server silently caps per_page at this value +const MAX_PER_PAGE = 10000 + export function computeJobKinds(jobKindsCat: string | null): string { if (jobKindsCat == 'all') { return '' @@ -75,6 +78,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { let label = $derived(filters?.label ?? null) let worker = $derived(filters?.worker ?? null) let success = $derived(filters?.status ?? null) + let isQueueOnly = $derived(success == 'running' || success == 'suspended' || success == 'waiting') let showSkipped = $derived(filters?.show_skipped ?? false) let resolutionFilter = $derived(filters?.resolved ?? 'all') let showSchedules = $derived(!filters?.job_trigger_kind?.includes('!schedule')) @@ -126,9 +130,14 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { let promise = loadJobsIntern(true) if (perPage > 25) { promise = CancelablePromiseUtils.onTimeout(promise, 4000, () => { - sendUserToast('Loading jobs is taking longer than expected...', 'warning', [ - { label: 'Stream by batches of 25', callback: () => restreamWithSmallBatches() } - ]) + const noStartDate = timeframe?.computeMinMax().minTs == null + sendUserToast( + (success == 'failure' || success == 'canceled') && noStartDate + ? `Loading ${success == 'failure' ? 'failed' : 'canceled'} jobs with no start date scans the full job history. Set a time range to speed it up.` + : 'Loading jobs is taking longer than expected...', + 'warning', + [{ label: 'Stream by batches of 25', callback: () => restreamWithSmallBatches() }] + ) }) } promise = CancelablePromiseUtils.finallyDo(promise, () => { @@ -191,21 +200,40 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loadingExtra = false } + // Mirrors when list_completed_jobs_query sorts by completed_at. A created_at cursor does not + // bound that index scan, so each batch would rescan from the newest job, and skip jobs created + // after the cursor but completed before it. + function sortsByCompletedAt(minTs: string | null, maxTs: string | null): boolean { + return minTs != null || maxTs != null || success == 'failure' || success == 'canceled' + } + function loadExtraJobsBatch(batchSize: number): CancelablePromise { if (!jobs || jobs.length === 0) { lastFetchWentToEnd = true return CancelablePromiseUtils.pure(undefined as void) } - const lastJob = jobs[jobs.length - 1] - const ts = lastJob.created_at - if (!ts) { + const { minTs, maxTs } = timeframe?.computeMinMax() ?? { minTs: null, maxTs: null } + const byCompletedAt = + jobs[jobs.length - 1].type === 'CompletedJob' && sortsByCompletedAt(minTs, maxTs) + const sortKey = (j: Job) => + byCompletedAt ? (j.type === 'CompletedJob' ? j.completed_at : undefined) : j.created_at + const cursorTs = sortKey(jobs[jobs.length - 1]) + if (!cursorTs) { lastFetchWentToEnd = true return CancelablePromiseUtils.pure(undefined as void) } - const cursorTs = new Date(new Date(ts).getTime() - 1).toISOString() - const minTs = timeframe?.computeMinMax().minTs ?? null + // Inclusive cursor at the API's microsecond precision: jobs sharing the boundary timestamp (e.g. + // a bulk cancel) are refetched rather than skipped, and the page grows by those already listed, + // up to the server's MAX_PER_PAGE. Once the listed part of the group fills that cap, the cursor + // steps just below the group, dropping its remainder instead of ending the list early. + const tied = jobs.filter((j) => sortKey(j) === cursorTs).length + const stepOver = tied >= MAX_PER_PAGE + const cursor = stepOver ? new Date(new Date(cursorTs).getTime() - 1).toISOString() : cursorTs + const pageSize = stepOver ? batchSize : Math.min(batchSize + tied, MAX_PER_PAGE) return CancelablePromiseUtils.map( - fetchJobs(null, minTs, undefined, cursorTs, batchSize), + byCompletedAt + ? fetchJobs(cursor, minTs, undefined, undefined, pageSize) + : fetchJobs(null, minTs, undefined, cursor, pageSize), (olderJobs) => { jobs = updateWithNewJobs(olderJobs ?? [], jobs ?? []) if (extendedJobs) { @@ -213,7 +241,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { extendedJobs = extendedJobs } computeCompletedJobs() - lastFetchWentToEnd = (olderJobs?.length ?? 0) < batchSize + lastFetchWentToEnd = (olderJobs?.length ?? 0) < pageSize loading = false } ) @@ -230,7 +258,6 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loadingFetch = true let scriptPathStart = folder == null || folder === '' ? undefined : `f/${folder}/` let scriptPathExact = path == null || path === '' ? undefined : path - let isQueueOnly = success == 'running' || success == 'suspended' || success == 'waiting' let isCompletedOnly = success == 'success' || success == 'failure' || success == 'canceled' let promise = JobService.listJobs({ workspace: currentWorkspace, @@ -289,7 +316,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { }) promise = CancelablePromiseUtils.catchErr(promise, (e) => { if (e instanceof CancelError) return CancelablePromiseUtils.err(e) - sendUserToast('There was an issue loading jobs, see browser console for more details', true) + sendUserToast(`Could not load jobs: ${e.body ?? e.message}`, true) console.error(e) return CancelablePromiseUtils.pure([]) }) @@ -394,6 +421,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { overrideBatchSize?: number ): CancelablePromise { const { minTs, maxTs } = timeframe?.computeMinMax() ?? { minTs: null, maxTs: null } + listLoadedAt = new Date(Date.now() - 5 * 60_000).toISOString() if (shouldGetCount) { getCount() } @@ -529,6 +557,7 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { } let lastQueueTs: string | undefined = undefined + let listLoadedAt: string | null = null async function syncer() { if (loadingFetch) { @@ -575,7 +604,15 @@ export function useJobsLoader(args: () => UseJobLoaderArgs) { loading = true let newJobs: Job[] if (concurrencyKey == null || concurrencyKey === '') { - newJobs = await fetchJobs(maxTs, minTs ?? completedTs, queueTs) + // With no completed job to anchor on, each refresh would repeat the initial + // unbounded scan of completed jobs, possibly the one that just timed out. Not for + // queue-only views: fetchJobs turns this into the queue's created_at bound, hiding + // older jobs that suspend or come due. The margin absorbs browser/database skew. + newJobs = await fetchJobs( + maxTs, + minTs ?? completedTs ?? (isQueueOnly ? null : listLoadedAt), + queueTs + ) } else { // Obscured jobs have no ids, so we have to do the full request extendedJobs = await fetchExtendedJobs(concurrencyKey, maxTs, minTs ?? completedTs) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index a117d2769c..b94e0faefa 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -29,6 +29,8 @@ WandSparkles } from 'lucide-svelte' import Portal from '$lib/components/Portal.svelte' + import { zIndexes } from '$lib/zIndexes' + import { overlayStack } from '$lib/components/common/overlayHost.svelte' import { twMerge } from 'tailwind-merge' import ContentSearchInner from '../ContentSearchInner.svelte' @@ -367,6 +369,7 @@ async function handleKeydown(event: KeyboardEvent) { if ((!isMac() ? event.ctrlKey : event.metaKey) && event.key === 'k') { event.preventDefault() + if (!open) openedOn = undefined await openModal() } if (open) { @@ -427,7 +430,7 @@ path = `/apps/get/${e.path}` break case 'raw_app': - path = `/raw_apps/get/${e.path}` + path = `/apps_raw/get/${e.path}` break default: path = '/' @@ -450,6 +453,24 @@ mouseMoved = true } + // On the overlay stack while open: a modal or drawer it was opened from arbitrates Escape on that + // stack, and would otherwise close itself on the key meant for the search above it. The opener + // names its stack, because a pane hosting an editor (a sessions tab) keeps its own. + const globalStack = overlayStack() + let openedOn: import('$lib/components/common/overlayHost.svelte').OverlayStack | undefined = + $state(undefined) + const STACK_ID = 'global-search' + $effect(() => { + if (!open) return + const stack = openedOn ?? globalStack + untrack(() => stack.val.push(STACK_ID)) + return () => { + untrack(() => { + stack.val = stack.val.filter((id) => id !== STACK_ID) + }) + } + }) + onMount(() => { window.addEventListener('keydown', handleKeydown) window.addEventListener('mousemove', handleMouseMove) @@ -559,7 +580,11 @@ } } - export async function openSearchWithPrefilledText(text?: string) { + export async function openSearchWithPrefilledText( + text?: string, + stack?: import('$lib/components/common/overlayHost.svelte').OverlayStack + ) { + openedOn = stack await openModal() searchTerm = text ?? searchTerm await handleSearch() @@ -618,9 +643,9 @@
({ + VariableService: { + createVariable: vi.fn(async ({ requestBody }: any) => { + created.push(requestBody) + return requestBody.path + }) + } +})) + +vi.mock('$lib/stores', async () => { + const { writable } = await import('svelte/store') + return { workspaceStore: writable('test-ws'), userStore: writable({ username: 'ada' }) } +}) + +import { processSecretArgs } from './secretArgUtils' + +describe('processSecretArgs', () => { + beforeEach(() => (created.length = 0)) + + const schema = { + properties: { + token: { type: 'string', password: true }, + creds: { type: 'object', password: true, properties: { user: { type: 'string' } } }, + nested: { type: 'object', properties: { inner: { type: 'string', password: true } } }, + plain: { type: 'string' } + } + } as any + + // Nothing else turns a proposed secret into a reference when no form mounts, so a literal + // left alone here is a plaintext credential stored on the job for anyone who can see it. + it('mints a reference for a literal at every level, leaving other arguments alone', async () => { + const out = await processSecretArgs( + { token: 'hunter2', creds: { user: 'ada' }, nested: { inner: 'deep' }, plain: 'kept' }, + schema + ) + expect(out.token).toMatch(/^\$var:u\/ada\/secret_arg\//) + expect(out.creds).toMatch(/^\$jsonvar:u\/ada\/secret_arg\//) + expect(out.nested.inner).toMatch(/^\$var:u\/ada\/secret_arg\//) + expect(out.plain).toBe('kept') + // The object goes into the variable as JSON, which is what `$jsonvar:` parses back. + expect(created.map((c) => c.value).sort()).toEqual(['deep', 'hunter2', '{"user":"ada"}']) + expect(created.every((c) => c.is_secret)).toBe(true) + }) + + it('leaves a reference the caller already named alone', async () => { + const out = await processSecretArgs( + { token: '$var:f/team/api_token', creds: '$jsonvar:u/ada/existing' }, + schema + ) + expect(out).toEqual({ token: '$var:f/team/api_token', creds: '$jsonvar:u/ada/existing' }) + expect(created).toEqual([]) + }) + + // `$var:` hands the job the variable's text; a field declaring an object needs it parsed, + // which is the same variable read the other way rather than a secret the caller cannot see. + it('reads a plain variable as JSON where the field cannot hold a string', async () => { + const out = await processSecretArgs({ creds: '$var:u/ada/stripe' }, schema) + expect(out.creds).toBe('$jsonvar:u/ada/stripe') + expect(created).toEqual([]) + }) + + // Reached by every run form in the product, not only the ones a chat opens. + it('leaves an absent, null or empty secret alone', async () => { + expect(await processSecretArgs({ token: null, plain: 'kept' }, schema)).toEqual({ + token: null, + plain: 'kept' + }) + expect(await processSecretArgs({ token: '', plain: 'kept' }, schema)).toEqual({ + token: '', + plain: 'kept' + }) + expect(created).toEqual([]) + }) + + // A property name can itself contain a dot. Reported under one label these two leaves + // would share a mint, and the flat field would run on the nested field's secret. + it("tells apart a key that spells another key's path", async () => { + const out = await processSecretArgs({ 'db.password': 'FLAT', db: { password: 'NESTED' } }, { + properties: { + 'db.password': { type: 'string', password: true }, + db: { + type: 'object', + properties: { password: { type: 'string', password: true } } + } + } + } as any) + const flat = out['db.password'].slice('$var:'.length) + const nested = out.db.password.slice('$var:'.length) + expect(flat).not.toBe(nested) + expect(created.find((c) => c.path === flat)?.value).toBe('FLAT') + expect(created.find((c) => c.path === nested)?.value).toBe('NESTED') + }) + + // Callers bind a form that stays editable while the mints are in flight, and a leaf is + // addressed by its path: a row moved between the two walks would take the other row's + // reference and the job would run it on the wrong credentials. + it('ignores the caller mutating the arguments while minting', async () => { + const rows = { + creds: [ + { name: 'alpha', secret: 'FIRST' }, + { name: 'beta', secret: 'SECOND' } + ] + } + const arraySchema = { + properties: { + creds: { + type: 'array', + items: { properties: { name: {}, secret: { password: true } } } + } + } + } as any + + const pending = processSecretArgs(rows, arraySchema) + rows.creds.reverse() + const out = await pending + + // Keyed by the row's own name, not its index: a substitution by position lands the + // first-minted reference on index 0 either way. + const secretOf = (name: string) => { + const row = out.creds.find((c: any) => c.name === name) + return created.find((c) => c.path === row.secret.slice('$var:'.length))?.value + } + expect(secretOf('alpha')).toBe('FIRST') + expect(secretOf('beta')).toBe('SECOND') + }) +}) diff --git a/frontend/src/lib/components/secretArgUtils.ts b/frontend/src/lib/components/secretArgUtils.ts index 91101d4816..a330d3bcd9 100644 --- a/frontend/src/lib/components/secretArgUtils.ts +++ b/frontend/src/lib/components/secretArgUtils.ts @@ -3,11 +3,54 @@ import { VariableService } from '$lib/gen' import { get } from 'svelte/store' import { userStore, workspaceStore } from '$lib/stores' import { generateRandomString } from '$lib/utils' +import { stateSnapshot } from '$lib/stateSnapshot.svelte' +import { isSecretProp, mapArgLeaves } from './job_args' + +/** Where a caller's own ephemeral secrets live, so a field can tell one it minted from a + * workspace variable someone linked by hand. */ +export function ephemeralSecretPrefix(username: string): string { + return `u/${username}/secret_arg/` +} /** - * Process args before job submission: for non-string fields marked as password/sensitive, - * create ephemeral secret variables and replace values with $jsonvar:path references. - * String password fields are already handled by PasswordArgInput (uses $var:). + * Mint the ephemeral secret variable a sensitive argument is submitted as, and return its path. + * It expires on its own, so a run that is abandoned leaves no permanent secret behind. + */ +export async function mintEphemeralSecret( + workspace: string, + username: string, + value: string +): Promise { + const path = ephemeralSecretPrefix(username) + generateRandomString(12) + await VariableService.createVariable({ + workspace, + requestBody: { + value, + is_secret: true, + path, + description: 'Ephemeral secret variable', + expires_at: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7).toISOString() + } + }) + return path +} + +/** `$var:` hands the job the variable's text and `$jsonvar:` hands it the parsed value, so a + * field that cannot hold a string needs the second one whichever the caller named. */ +function referencePrefix(prop: any, value: unknown): '$var:' | '$jsonvar:' { + return typeof value === 'string' && prop?.type !== 'object' && prop?.type !== 'array' + ? '$var:' + : '$jsonvar:' +} + +/** + * Turn every sensitive argument into a reference before the job is submitted: a plaintext value + * is minted into an ephemeral secret variable, so what is stored on the job — and readable by + * anyone who can see its run — names a secret instead of holding one. + * + * The single place that decides how a secret reaches a job: {@link PasswordArgInput} mints + * through it while the user types, and a run the autonomy posture starts without a form calls it + * in the widget's stead. */ export async function processSecretArgs( args: Record, @@ -24,29 +67,53 @@ export async function processSecretArgs( const username = (user.username ?? user.email)?.split('@')[0] if (!username) return args - const userPrefix = `u/${username}/secret_arg/` - const result = { ...args } + // Detached from the caller: every one binds a form that stays editable across the awaits + // below, and the two walks address a leaf by its path — an array reordered between them + // would hand a row the reference minted for another row's secret. + args = stateSnapshot(args) - for (const [key, prop] of Object.entries(schema.properties)) { - if (!prop.password) continue - if (prop.type !== 'object') continue // only object types; strings handled by PasswordArgInput - if (result[key] == null || result[key] === undefined) continue - if (typeof result[key] === 'string' && result[key].startsWith('$jsonvar:')) continue // already processed + // A value that already names a variable is one; anything else is the secret itself. An empty + // field holds nothing to mint, and ArgInput synthesises '' for every untouched string. + const holdsSecret = (value: unknown) => + value != null && + value !== '' && + !( + typeof value === 'string' && + (value.startsWith('$var:') || value.startsWith('$jsonvar:') || value.startsWith('$res:')) + ) - const path = userPrefix + generateRandomString(12) - await VariableService.createVariable({ + // Collected first and substituted after, because the walk is synchronous and minting is not. + // Keyed by the whole path the walk reports, which is what tells two same-named leaves apart. + const pending: { key: string; prop: any; value: unknown }[] = [] + mapArgLeaves(args, schema as any, isSecretProp, (value, prop, path) => { + if (holdsSecret(value)) pending.push({ key: JSON.stringify(path), prop, value }) + return value + }) + + const minted = new Map() + for (const { key, prop, value } of pending) { + const reference = referencePrefix(prop, value) + const variable = await mintEphemeralSecret( workspace, - requestBody: { - value: JSON.stringify(result[key]), - is_secret: true, - path, - description: 'Ephemeral secret variable', - expires_at: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7).toISOString() - } - }) - result[key] = '$jsonvar:' + path + username, + reference === '$var:' ? String(value) : JSON.stringify(value) + ) + minted.set(key, reference + variable) } - return result + return mapArgLeaves(args, schema as any, isSecretProp, (value, prop, path) => { + const replacement = minted.get(JSON.stringify(path)) + if (replacement !== undefined) return replacement + // A plain variable named for a field that cannot hold a string: the caller meant that + // variable's contents, which is the same secret read the way the field needs it. + if ( + typeof value === 'string' && + value.startsWith('$var:') && + referencePrefix(prop, value) === '$jsonvar:' + ) { + return '$jsonvar:' + value.slice('$var:'.length) + } + return value + }) } diff --git a/frontend/src/lib/components/sessions/ItemViewerView.svelte b/frontend/src/lib/components/sessions/ItemViewerView.svelte index f84de14cd0..56652a1d2a 100644 --- a/frontend/src/lib/components/sessions/ItemViewerView.svelte +++ b/frontend/src/lib/components/sessions/ItemViewerView.svelte @@ -176,7 +176,6 @@ onNavigate(`/apps_raw/edit/${path}`)} onLoadState={setLoadState} /> diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index d361a2f257..138124a559 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -19,6 +19,7 @@ } from './previewRouter' import { withMenuHidden } from './sessionMode.svelte' import ArtifactViewer from '../copilot/chat/artifacts/ArtifactViewer.svelte' + import RunFormPreviewSlot from './RunFormPreviewSlot.svelte' import { setOverlayHost } from '../common/overlayHost.svelte' let { @@ -416,6 +417,20 @@
This artifact is no longer available.
{/if}
+{:else if slot.kind === 'runform' && mounted} +
+ + {#if runtime && overlayHostEl} + + {/if} +
{:else if mounted}
+ {#if item.leadsCatalogue} +

+ More from the hub +

+ {/if} + +