diff --git a/.agents/skills/local-review-codex/SKILL.md b/.agents/skills/local-review-codex/SKILL.md index cc932f7a4b..ec1277fe3a 100644 --- a/.agents/skills/local-review-codex/SKILL.md +++ b/.agents/skills/local-review-codex/SKILL.md @@ -1,6 +1,6 @@ --- name: local-review-codex -description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy, model, and reasoning effort as the codex-pr-review GitHub action. +description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy and reasoning effort as the codex-pr-review GitHub action, on a newer model. --- # Local Codex Review (pre-push) @@ -11,17 +11,18 @@ before the PR exists. Use this before `git push` on a non-trivial change. **Correspondence with CI** — identical: - Policy: `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test coverage). -- Model: `gpt-5.6-sol`, `model_reasoning_effort="xhigh"`. +- Reasoning effort: `model_reasoning_effort="xhigh"`. - Output: markdown starting with `## Codex Review`, findings tagged P0 / P1 / P2 with file:line. **Differences from CI** — local-only: +- Model is `gpt-6-astra`; CI stays on `gpt-5.6-sol`. Not an oversight to reconcile: `gpt-6-astra` is confirmed on the ChatGPT auth `codex login` uses locally, while CI authenticates with `OPENAI_API_KEY` (`codex-pr-review.yml` prefers it over `CODEX_AUTH_JSON`) and that tier is unverified for the model. Move CI once API access is confirmed, or once CI switches to `CODEX_AUTH_JSON`. - Scope is the current branch vs `main` at the merge-base, **including uncommitted changes** (CI reviews a pushed PR diff). - Sandbox is `read-only` (CI uses `danger-full-access` on an ephemeral runner). Codex reads the diff and files but cannot modify your working tree. - Fresh context is inherent: `codex exec` is a separate cold process, so it does not anchor on the current chat session — the same reason `local-review` insists on a subagent. ## Prerequisites -- `codex` CLI **>= 0.144.1** installed and authed (`codex login` or `OPENAI_API_KEY`). Older CLIs reject `gpt-5.6-sol` with "requires a newer version of Codex". Upgrade with `npm install --global @openai/codex@0.144.1` (may need `sudo` for a global install). Keep this in sync with the pin in `.github/workflows/codex-pr-review.yml`. +- `codex` CLI **>= 0.153.4** installed and authed via `codex login` (an `OPENAI_API_KEY` in the environment takes priority and may not reach `gpt-6-astra` — see the model note above). Older CLIs reject the model with "requires a newer version of Codex"; `run.sh` checks the version up front. Upgrade with `npm install --global @openai/codex@0.153.4` (may need `sudo` for a global install). This matches the pin in `.github/workflows/codex-pr-review.yml` — the CLI version is the same on both sides, only the model differs. - `git fetch` the base ref if it's stale, so the merge-base is accurate. ## Run diff --git a/.agents/skills/local-review-codex/run.sh b/.agents/skills/local-review-codex/run.sh index d6491099c2..948d3820eb 100755 --- a/.agents/skills/local-review-codex/run.sh +++ b/.agents/skills/local-review-codex/run.sh @@ -1,21 +1,44 @@ #!/usr/bin/env bash # Local Codex review — mirrors the .github/workflows/codex-pr-review.yml CI job, # but scoped to this branch's unpushed work (committed + uncommitted) so you can -# review before pushing. Same policy (REVIEW.md), same model (gpt-5.6-sol) and -# reasoning effort (xhigh) as CI. Runs read-only: Codex cannot modify your tree. +# review before pushing. Same policy (REVIEW.md) and reasoning effort (xhigh) as CI. +# +# The model deliberately differs from CI: gpt-6-astra is confirmed available on the +# ChatGPT auth `codex login` uses here, but CI authenticates with OPENAI_API_KEY and +# that tier is unverified for it, so codex-pr-review.yml stays on gpt-5.6-sol. # # Usage: run.sh [BASE_REF] (BASE_REF defaults to "main") set -euo pipefail +MODEL="gpt-6-astra" +CODEX_MIN="0.153.4" + BASE_REF="${1:-main}" REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" if ! command -v codex >/dev/null 2>&1; then - echo "codex CLI not found. Install with: npm install --global @openai/codex@0.144.1" >&2 + echo "codex CLI not found. Install with: npm install --global @openai/codex@$CODEX_MIN" >&2 exit 1 fi +# Older CLIs reject the model with an error that never names the CLI version as the +# cause, so check it up front rather than letting the exec fail opaquely. The `|| true` +# keeps an unrecognised --version format from aborting under `set -e`: an unparseable +# version means "cannot tell", which must fall through to the exec, not kill the review. +CODEX_VER="$(codex --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" +if [ -n "$CODEX_VER" ] && [ "$(printf '%s\n%s\n' "$CODEX_MIN" "$CODEX_VER" | sort -V | head -1)" != "$CODEX_MIN" ]; then + echo "codex $CODEX_VER is too old for $MODEL (need >= $CODEX_MIN). Upgrade with: npm install --global @openai/codex@$CODEX_MIN" >&2 + exit 1 +fi + +# codex prefers OPENAI_API_KEY over the ChatGPT credentials `codex login` stores, and +# that tier is not confirmed for $MODEL — the resulting failure names the model, not the +# auth that selected it. +if [ -n "${OPENAI_API_KEY:-}" ]; then + echo "warning: OPENAI_API_KEY is set and takes priority over 'codex login' credentials; $MODEL may be unavailable on that tier." >&2 +fi + # Resolve the base to a concrete commit, preferring a local ref but falling back to # the remote-tracking ref — checkouts (CI, single-branch clones) often have only # origin/main, not a local main. @@ -80,7 +103,7 @@ EOF codex exec \ -C "$REPO_ROOT" \ - -m gpt-5.6-sol \ + -m "$MODEL" \ -c 'model_reasoning_effort="xhigh"' \ -s read-only \ -o "$OUT" \ diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 26b2d9aae8..7a3882df7c 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -219,7 +219,7 @@ jobs: - name: Install Codex CLI if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - run: npm install --global @openai/codex@0.144.1 + run: npm install --global @openai/codex@0.153.4 - name: Configure Codex auth if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5dc2004c6f..f4fac9d2e0 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.804.0" + ".": "1.805.0" } diff --git a/AGENTS.md b/AGENTS.md index f1ef0bd95f..f8c83ec465 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Open-source platform for internal tools, workflows, API integrations, background - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. - **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead. -- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy, `gpt-5.6-sol`, `xhigh` reasoning; requires the `codex` CLI >= 0.144.1. +- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy and `xhigh` reasoning, on `gpt-6-astra` rather than the action's `gpt-5.6-sol`; requires the `codex` CLI >= 0.153.4. - **Domain guides**: `.claude/skills/native-trigger/` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfb0edad..f431b9fd96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) + + +### Features + +* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9)) +* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63)) +* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c)) +* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4)) +* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a)) +* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab)) + + +### Bug Fixes + +* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728)) +* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198)) +* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43)) +* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c)) +* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac)) +* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de)) +* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f)) +* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f)) +* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13)) + ## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05) diff --git a/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json new file mode 100644 index 0000000000..db9d674fcf --- /dev/null +++ b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock',\n 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily',\n 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d" +} diff --git a/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json new file mode 100644 index 0000000000..00b839de6a --- /dev/null +++ b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf" +} diff --git a/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json new file mode 100644 index 0000000000..7bb2cac5db --- /dev/null +++ b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6" +} diff --git a/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json new file mode 100644 index 0000000000..b60dd43fdf --- /dev/null +++ b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6" +} diff --git a/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json new file mode 100644 index 0000000000..77b00d9daa --- /dev/null +++ b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704" +} diff --git a/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json new file mode 100644 index 0000000000..bdb54cfbb9 --- /dev/null +++ b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c" +} diff --git a/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json new file mode 100644 index 0000000000..50755132fc --- /dev/null +++ b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4,\n 'source', $4, $5, '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6,\n 'model', $6, $7, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c" +} diff --git a/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json new file mode 100644 index 0000000000..7148252002 --- /dev/null +++ b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188" +} diff --git a/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json new file mode 100644 index 0000000000..fbfab14017 --- /dev/null +++ b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db" +} diff --git a/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json new file mode 100644 index 0000000000..28b34afa97 --- /dev/null +++ b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind,\n ingested_at)\n SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column,\n child_unique_id, child_column, lineage_kind, ingested_at\n FROM dbt_column_edge\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4" +} diff --git a/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json new file mode 100644 index 0000000000..1268656e34 --- /dev/null +++ b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest, manifest_key, run_results,\n run_results_key, updated_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET\n job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest,\n manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results,\n run_results_key = EXCLUDED.run_results_key, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Uuid", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3" +} diff --git a/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json new file mode 100644 index 0000000000..7f950346ee --- /dev/null +++ b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4" +} diff --git a/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json new file mode 100644 index 0000000000..24e563d9ad --- /dev/null +++ b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "edge!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e" +} diff --git a/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json new file mode 100644 index 0000000000..913c16c31d --- /dev/null +++ b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text\n WHERE EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false\n AND language = 'dbt'\n AND (hash = $4 OR $4 = ANY(parent_hashes)))\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0" +} diff --git a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json similarity index 70% rename from backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json rename to backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json index da77dc1de5..134e58ed58 100644 --- a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json +++ b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", + "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.column_schema, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", "describe": { "columns": [ { @@ -80,21 +80,26 @@ }, { "ordinal": 15, - "name": "freshness", + "name": "column_schema", "type_info": "Jsonb" }, { "ordinal": 16, + "name": "freshness", + "type_info": "Jsonb" + }, + { + "ordinal": 17, "name": "raw_code", "type_info": "Text" }, { - "ordinal": 17, + "ordinal": 18, "name": "original_file_path", "type_info": "Text" }, { - "ordinal": 18, + "ordinal": 19, "name": "script_visible!", "type_info": "Bool" } @@ -127,8 +132,9 @@ true, true, true, + true, null ] }, - "hash": "9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12" + "hash": "4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5" } diff --git a/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json new file mode 100644 index 0000000000..21759e2fe5 --- /dev/null +++ b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT p.asset_path AS \"from_path!\", e.parent_column AS \"from_column!\",\n c.asset_path AS \"to_path!\", e.child_column AS \"to_column!\",\n e.lineage_kind AS \"kind!\"\n FROM unnest($2::text[], $3::bigint[], $4::uuid[])\n AS o(script_path, script_hash, job_id)\n JOIN dbt_column_edge e ON e.workspace_id = $1\n AND e.script_path = o.script_path\n AND e.job_id = o.job_id\n -- `=` still, with the NULL-to-NULL case\n -- spelled out and gated on the pin: a\n -- version-less row's hash is NULL on both\n -- sides, which `=` never matches, but\n -- `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound everywhere else.\n AND (e.script_hash = o.script_hash\n OR ($5::text IS NOT NULL\n AND o.script_hash IS NULL\n AND e.script_hash IS NULL))\n JOIN dbt_node p ON p.workspace_id = e.workspace_id\n AND p.script_path = e.script_path\n AND p.script_hash IS NOT DISTINCT FROM e.script_hash\n AND p.job_id = e.job_id\n AND p.unique_id = e.parent_unique_id\n JOIN dbt_node c ON c.workspace_id = e.workspace_id\n AND c.script_path = e.script_path\n AND c.script_hash IS NOT DISTINCT FROM e.script_hash\n AND c.job_id = e.job_id\n AND c.unique_id = e.child_unique_id\n WHERE e.lineage_kind IN ('copy', 'mod')\n AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "from_path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "from_column!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "to_path!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "to_column!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array", + "UuidArray", + "Text" + ] + }, + "nullable": [ + true, + false, + true, + false, + false + ] + }, + "hash": "55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d" +} diff --git a/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json new file mode 100644 index 0000000000..822695a6c9 --- /dev/null +++ b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms)\n VALUES ($1, $2, $2, '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d" +} diff --git a/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json new file mode 100644 index 0000000000..b2a0d728cd --- /dev/null +++ b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false AND language = 'dbt'\n AND (hash = $3 OR $3 = ANY(parent_hashes))\n FOR SHARE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147" +} diff --git a/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json new file mode 100644 index 0000000000..09efdc7340 --- /dev/null +++ b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521" +} diff --git a/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json new file mode 100644 index 0000000000..8b04c9e258 --- /dev/null +++ b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10" +} diff --git a/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json new file mode 100644 index 0000000000..07c923620a --- /dev/null +++ b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7" +} diff --git a/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json new file mode 100644 index 0000000000..b1b92ecc75 --- /dev/null +++ b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "resource_type", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0" +} diff --git a/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json new file mode 100644 index 0000000000..3a67641732 --- /dev/null +++ b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 6000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837" +} diff --git a/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json new file mode 100644 index 0000000000..c5740cfb88 --- /dev/null +++ b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_environment_state SET script_path = $3\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba" +} diff --git a/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json new file mode 100644 index 0000000000..f4bd63c718 --- /dev/null +++ b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630" +} diff --git a/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json new file mode 100644 index 0000000000..a498ecccd0 --- /dev/null +++ b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912" +} diff --git a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json deleted file mode 100644 index 823a65cda3..0000000000 --- a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47" -} diff --git a/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json new file mode 100644 index 0000000000..b98a71fb8a --- /dev/null +++ b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26" +} diff --git a/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json new file mode 100644 index 0000000000..49d01c2b8b --- /dev/null +++ b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders',\n 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders',\n 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82" +} diff --git a/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json new file mode 100644 index 0000000000..909e6ad42d --- /dev/null +++ b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247" +} diff --git a/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json new file mode 100644 index 0000000000..2e0e072867 --- /dev/null +++ b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf" +} diff --git a/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json new file mode 100644 index 0000000000..6b2ec01b23 --- /dev/null +++ b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946" +} diff --git a/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json new file mode 100644 index 0000000000..7476ae9871 --- /dev/null +++ b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2\n AND NOT EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64" +} diff --git a/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json new file mode 100644 index 0000000000..934742008f --- /dev/null +++ b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders',\n 'u/a/wh/analytics/raw_orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24" +} diff --git a/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json new file mode 100644 index 0000000000..fee6801a6a --- /dev/null +++ b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n 'u/a/wh/analytics/draft', 'select 3', '{}'),\n ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src',\n 'u/a/wh/analytics/draft_src', 'select 4', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f" +} diff --git a/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json new file mode 100644 index 0000000000..9c75f7ac97 --- /dev/null +++ b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c" +} diff --git a/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json new file mode 100644 index 0000000000..748ca6d9ec --- /dev/null +++ b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac" +} diff --git a/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json new file mode 100644 index 0000000000..46a9c8e505 --- /dev/null +++ b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570" +} diff --git a/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json new file mode 100644 index 0000000000..797fee956b --- /dev/null +++ b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77" +} diff --git a/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json new file mode 100644 index 0000000000..854ed61857 --- /dev/null +++ b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT n.script_path AS \"script_path!\", n.script_hash, n.job_id AS \"job_id!\"\n FROM dbt_node n\n WHERE n.workspace_id = $1 AND n.asset_path = ANY($2)\n -- The run's snapshot, or the deployed graph when that job stored\n -- none -- a build pins only if it wrote one.\n AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $5)\n THEN $5::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END\n -- The gate, re-decided for every project the walk reaches. That\n -- is what resolving owners in a loop is for: being entitled to\n -- one project is not being entitled to the one that declares a\n -- relation it hands over.\n AND ( $6\n OR n.script_path = ANY($7)\n OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx\n WHERE n.script_path = pfx\n OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) )\n AND CASE\n -- Pinned: which version comes from a job this caller was\n -- already granted, so `script` does not decide THAT -- but\n -- it still decides whether the project may be read, the\n -- same second gate `script_visible` is on the graph. Being\n -- entitled to a run is not being entitled to the SQL\n -- behind it, and column lineage is that SQL's shape. A\n -- version-less row is exempt because it is an editor\n -- buffer, which has no `script` row to ask and reaches\n -- this only through the parse job that wrote it.\n --\n -- One project answers, so a pinned trace never crosses\n -- into another: neither does the graph it annotates.\n WHEN $4::text IS NOT NULL\n THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint\n AND ($3::bigint IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3))\n -- A named version: the deployed one an editor is drawing.\n -- `script` is read under RLS, so this is the visibility\n -- check as well as the existence one. A hash names one\n -- script row, so this arm answers for one project too —\n -- and deliberately: a pin says which stored graph is on\n -- screen, and another project's live graph is not it.\n WHEN $3::bigint IS NOT NULL\n THEN n.script_hash = $3 AND EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3)\n -- Otherwise the version deployed now: an older one's rows\n -- outlive it in `dbt_node` until the sweep, and describe a\n -- project that is no longer what runs. `language` narrows\n -- it the way the graph's own resolution does, so a path\n -- that has since become a script of another kind draws and\n -- explains the same version rather than disagreeing. Read\n -- under RLS, so a project the caller cannot see resolves\n -- to NULL and matches nothing.\n ELSE n.script_hash = (\n SELECT sc.hash FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.language = 'dbt'\n AND sc.deleted = false AND sc.archived = false\n ORDER BY sc.created_at DESC LIMIT 1)\n END", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "job_id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8", + "Text", + "Uuid", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2" +} diff --git a/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json new file mode 100644 index 0000000000..75526d2db8 --- /dev/null +++ b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b" +} diff --git a/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json new file mode 100644 index 0000000000..1c79a53ab0 --- /dev/null +++ b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14" +} diff --git a/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json new file mode 100644 index 0000000000..ccb9863853 --- /dev/null +++ b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id',\n 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e" +} diff --git a/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json new file mode 100644 index 0000000000..540d2e0e08 --- /dev/null +++ b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2" +} diff --git a/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json new file mode 100644 index 0000000000..c1eb5fa826 --- /dev/null +++ b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e4dc875e49..5a19b73d1c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2763,18 +2763,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2782,27 +2782,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm_winapi" @@ -6640,9 +6640,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ "bitflags 2.13.1", "cfg-if", @@ -6664,9 +6664,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "ipnetwork" @@ -9077,9 +9077,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" dependencies = [ "memchr", "ucd-trie", @@ -9087,9 +9087,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" dependencies = [ "pest", "pest_generator", @@ -9097,9 +9097,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" dependencies = [ "pest", "pest_meta", @@ -9110,9 +9110,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" dependencies = [ "pest", ] @@ -14747,7 +14747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -14835,7 +14835,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.804.0" +version = "1.805.0" dependencies = [ "async-stream", "async-trait", @@ -14868,7 +14868,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "argon2", @@ -15021,7 +15021,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15044,7 +15044,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15061,7 +15061,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15087,7 +15087,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.804.0" +version = "1.805.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15097,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15114,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15159,7 +15159,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15175,7 +15175,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15197,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15218,7 +15218,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15232,7 +15232,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -15267,7 +15267,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15292,7 +15292,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15320,7 +15320,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15342,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15362,7 +15362,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15428,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.804.0" +version = "1.805.0" dependencies = [ "lazy_static", "serde", @@ -15440,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.804.0" +version = "1.805.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15464,7 +15464,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15478,7 +15478,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15513,7 +15513,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "lazy_static", @@ -15527,7 +15527,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15546,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15652,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "futures", @@ -15672,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.804.0" +version = "1.805.0" dependencies = [ "regex", "serde", @@ -15687,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15714,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15731,7 +15731,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15747,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15768,7 +15768,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15799,7 +15799,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "arc-swap", @@ -15824,7 +15824,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-stream", @@ -15858,7 +15858,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15876,7 +15876,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15885,7 +15885,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15897,7 +15897,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15909,7 +15909,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -15921,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15933,7 +15933,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15945,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -15956,7 +15956,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15967,7 +15967,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15979,7 +15979,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15990,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16012,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -16024,7 +16024,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16038,7 +16038,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16055,7 +16055,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16068,7 +16068,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -16080,7 +16080,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16098,7 +16098,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16114,7 +16114,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16130,7 +16130,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16144,7 +16144,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16183,7 +16183,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "const_format", @@ -16223,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16234,7 +16234,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16269,7 +16269,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16293,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16326,7 +16326,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16353,7 +16353,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16386,7 +16386,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16406,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16440,7 +16440,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16476,7 +16476,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16499,7 +16499,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16523,7 +16523,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -16547,7 +16547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16582,7 +16582,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16610,7 +16610,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16635,7 +16635,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16654,7 +16654,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-once-cell", @@ -16699,6 +16699,7 @@ dependencies = [ "opentelemetry 0.30.0", "opentelemetry-proto 0.30.0", "oracle", + "parquet", "pem 3.0.6", "pep440_rs", "postgres-native-tls 0.5.3", @@ -16771,7 +16772,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.804.0" +version = "1.805.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7abb3b92ee..9ef1d099ee 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.804.0" +version = "1.805.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.804.0" +version = "1.805.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -665,6 +665,12 @@ process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" datafusion = "47.0.0" +# The row API only: a dbt engine's parquet index is six string columns, so this +# needs no arrow and no writer. `parquet` is already in the tree with `arrow` for +# every shipped edition (`oss_core`), and cargo unifies the features there; this +# set is what a build WITHOUT object storage compiles. ZSTD is what the engine +# writes today, snap what parquet writers most often default to. +parquet = { version = "55.2.0", default-features = false, features = ["snap", "zstd"] } object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" diff --git a/backend/migrations/20260904135713_dbt_environment_state.down.sql b/backend/migrations/20260904135713_dbt_environment_state.down.sql new file mode 100644 index 0000000000..dab9025417 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS dbt_environment_state; diff --git a/backend/migrations/20260904135713_dbt_environment_state.up.sql b/backend/migrations/20260904135713_dbt_environment_state.up.sql new file mode 100644 index 0000000000..a49df5b3d1 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.up.sql @@ -0,0 +1,52 @@ +-- The dbt state one project last built into one environment: the `manifest.json` +-- (and the `run_results.json` beside it) that `dbt --defer --state ` resolves +-- an unbuilt `ref()` through. +-- +-- Separate from `dbt_run_state`, which answers a different question. That one is +-- keyed by the executing principal and holds the LAST run whatever its outcome, +-- so `dbt retry` can resume its failures; this one is keyed by environment and +-- holds the last SUCCESSFUL run, because a relation a later run defers to has to +-- exist. Merging them would make a retry resume a run that is not the last one, +-- or a deferral point at relations a failed run never wrote. +CREATE TABLE IF NOT EXISTS dbt_environment_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + -- The workspace warehouse, the dbt target, and the database and schema that + -- target resolves to. All four, because deferring is resolving a relation + -- NAME: a repointed warehouse or a moved schema makes the stored manifest + -- describe relations that are not where this run would look for them, and the + -- run has no other way to notice. A move therefore reads as an environment + -- with no state yet rather than as state that silently no longer fits. + -- + -- TEXT rather than VARCHAR(255): a project bringing its own `profiles.yml` + -- spells its own schema and database, so the length is the project's. + environment TEXT NOT NULL, + -- The run that published it, so a deferring run can say what it deferred to. + job_id UUID NOT NULL, + -- Exactly one home each. A manifest grows with the project and passes a few + -- hundred KB on a handful of models, so a large one goes to the INSTANCE's + -- object storage and this row keeps the key; a small one stays here, where it + -- costs no round trip and works on an instance that has configured no storage + -- at all. The instance's and not the workspace's, because a member can write + -- the workspace bucket under a key of their choosing, and a manifest is what a + -- later run resolves every unbuilt `ref()` through. `run_results.json` is a + -- tenth of the size and takes the same two homes rather than a rule of its own. + manifest TEXT, + manifest_key TEXT, + run_results TEXT, + run_results_key TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, environment), + CONSTRAINT dbt_environment_state_manifest_one_home + CHECK (num_nonnulls(manifest, manifest_key) = 1), + CONSTRAINT dbt_environment_state_run_results_one_home + CHECK (num_nonnulls(run_results, run_results_key) <= 1) +); + +-- No age sweep, unlike the per-run graph rows next door: this table holds one +-- row per script per environment and replaces it in place, so it does not grow +-- with runs, and its reader is every later run of that script — a project that +-- runs monthly must still find last month's state. It goes with the script +-- instead, alongside `dbt_run_state`. +GRANT ALL ON dbt_environment_state TO windmill_user; +GRANT ALL ON dbt_environment_state TO windmill_admin; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.down.sql b/backend/migrations/20260904143633_dbt_column_lineage.down.sql new file mode 100644 index 0000000000..a053a94291 --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS dbt_column_edge; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS column_schema; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.up.sql b/backend/migrations/20260904143633_dbt_column_lineage.up.sql new file mode 100644 index 0000000000..cb463c249f --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.up.sql @@ -0,0 +1,68 @@ +-- Column-level lineage, from the engine's own static analysis. +-- +-- `manifest.json` carries none, which is why decision 14 recorded the feature as +-- unavailable. The edges exist in a different artifact: an engine that does +-- static analysis writes `target/index/dbt.column_lineage.parquet` under +-- `dbt compile --static-analysis strict --write-index`. That pass is opt-in per +-- project (`column_lineage: true`), because strict analysis rejects SQL the +-- default accepts and must never become a silent requirement of running a build. + +-- One column-to-column edge, keyed exactly like `dbt_edge`: a version's graph +-- dies with its version through the composite foreign key, a run's snapshot is +-- keyed by `job_id` with the zero UUID meaning "the version's own graph", and an +-- editor buffer's parse carries a NULL `script_hash` keyed to its preview job. +CREATE TABLE IF NOT EXISTS dbt_column_edge ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + parent_unique_id TEXT NOT NULL, + parent_column TEXT NOT NULL, + child_unique_id TEXT NOT NULL, + child_column TEXT NOT NULL, + -- dbt's own word for how the value travelled: `copy` (passthrough), `mod` + -- (transformed), `scan` (the column was read to produce the ROW rather than + -- the value -- a join key, a `where` predicate, a `group by`). TEXT rather + -- than an enum because the engine treats the set as open: its own reader maps + -- those three and returns anything else verbatim. + lineage_kind TEXT NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Two partial unique indexes rather than a primary key, for the reason + -- 20260801121717 gives: a versioned graph is keyed by its version, a buffer + -- parse by its job alone. `lineage_kind` is part of the key because it is part + -- of the fact: a column that is both projected and used as a predicate for the + -- same output column has a `copy` edge AND a `scan` one, and the digest counts + -- both. Leaving it out let `ON CONFLICT DO NOTHING` drop the second while the + -- digest still claimed it was stored. + CONSTRAINT dbt_column_edge_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_versioned_key + ON dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_editor_key + ON dbt_column_edge (workspace_id, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NULL; + +-- Same age sweep as the other per-run rows, and the same reason there is no +-- foreign key to `v2_job`. +CREATE INDEX IF NOT EXISTS idx_dbt_column_edge_run_age ON dbt_column_edge (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; + +-- The real column schema of a node, which only static analysis knows: an +-- ordered `[{"name": …, "type": …}]`, from `dbt.node_columns.parquet`. +-- +-- Beside `columns` rather than folded into it. `columns` is the DECLARED +-- metadata `manifest.json` carries -- the names an author wrote in `schema.yml` +-- and the prose against them -- and stays exactly that, so a project that +-- documents two of forty columns keeps saying so. This is the other forty, +-- typed, in the order the model produces them. +ALTER TABLE dbt_node ADD COLUMN IF NOT EXISTS column_schema JSONB; + +GRANT ALL ON dbt_column_edge TO windmill_user; +GRANT ALL ON dbt_column_edge TO windmill_admin; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 9ed900214c..13597b06c9 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index cbb7102820..7de860c637 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.804.0" +version = "1.805.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs index 24f1e2c39d..36617c5639 100644 --- a/backend/parsers/windmill-parser-yaml/src/dbt.rs +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -58,6 +58,31 @@ impl DbtEngine { pub fn emits_node_events(&self) -> bool { matches!(self, DbtEngine::DbtCore1x) } + + /// Whether the engine's CLI has `--write-index`, the flag that writes the + /// parquet index column lineage lives in. False for 1.x, whose Python CLI + /// has no such option. + /// + /// True is not a promise that the artifact appears: `dbt-core` 2.0.0-alpha.5 + /// accepts the flag, declares the views over `dbt.column_lineage` in its own + /// `views.sql`, and writes neither that parquet nor `dbt.node_columns`. Only + /// Fusion does today. Attempting the pass on both is what lets a later 2.x + /// release pick the feature up with no change here. + pub fn writes_column_index(&self) -> bool { + !matches!(self, DbtEngine::DbtCore1x) + } + + /// Whether the engine has `--defer-state`, the deferral-only half of + /// `--state`. + /// + /// It matters on one command. `dbt retry` reads the run it resumes from + /// `--state`, so an engine with only that flag cannot be told to defer and + /// to resume from the job's own results at once: handed the deferral's + /// directory, it resumes the all-green run stored there and rebuilds + /// nothing. Only dbt-core 1.x separates the two. + pub fn has_defer_state_flag(&self) -> bool { + matches!(self, DbtEngine::DbtCore1x) + } } /// How the warehouse connection is supplied. Both paths are supported @@ -127,6 +152,18 @@ pub struct DbtDescriptor { pub selector: Option, #[serde(default)] pub test_behavior: DbtTestBehavior, + /// Ingest column-to-column lineage and the real column schemas, from the + /// engine's static analysis. + /// + /// Opt-in, and it has to be: the artifact only appears under + /// `--static-analysis strict`, which rejects SQL the default accepts (an + /// unresolvable identifier is an error there and compiles fine otherwise). + /// Turning it on for everyone would make a stricter dialect the price of + /// deploying a dbt project. It is a separate `dbt compile` pass, so nothing + /// it decides can change what a build does; a project it cannot analyze + /// keeps the graph it has today. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub column_lineage: bool, /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are /// all normal — so values keep their YAML type; only string leaves carry /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing @@ -137,6 +174,16 @@ pub struct DbtDescriptor { pub threads: Option, #[serde(default)] pub full_refresh: bool, + /// Resolve a `ref()` a run does not build through the state the last + /// successful run of this environment published, rather than through the + /// schema that run writes into. + /// + /// Only the default for the `build` block's own `defer`, since the choice is + /// per run: the run that publishes an environment's state and the run that + /// defers to it are two invocations of ONE script (decision 6), so a project + /// that could only defer by descriptor could never populate what it reads. + #[serde(default)] + pub defer: bool, /// Automatic in-job retry of the nodes a build failed on. /// /// dbt already confines a failure to its own subtree, and `dbt retry` @@ -258,6 +305,7 @@ pub const RESERVED_ARG_NAMES: &[&str] = &[ "exclude", "vars", "full_refresh", + "defer", "dbt_command", "dbt_retry_job", "model", @@ -345,15 +393,26 @@ fn command_variants(d: &DbtDescriptor) -> Vec<(&'static str, Vec)> { "build", selection() .into_iter() - .chain([Arg { - name: "full_refresh".to_string(), - otyp: None, - typ: Typ::Bool, - has_default: true, - default: Some(serde_json::json!(d.full_refresh)), - oidx: None, - otyp_inferred: false, - }]) + .chain([ + Arg { + name: "full_refresh".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.full_refresh)), + oidx: None, + otyp_inferred: false, + }, + Arg { + name: "defer".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.defer)), + oidx: None, + otyp_inferred: false, + }, + ]) .collect(), ), ( @@ -546,7 +605,9 @@ fn property_of(arg: &Arg) -> serde_json::Value { ), "select" => Some( "dbt selection syntax, e.g. `tag:nightly`, `stg_orders+`, \ - `config.materialized:incremental`. Empty runs the descriptor's own selection.", + `config.materialized:incremental`. `state:modified+` and `result:error+` \ + compare against the state a previous run published, so they need `defer` on. \ + Empty runs the descriptor's own selection.", ), "exclude" => Some("Nodes to leave out of the selection above, same syntax."), "vars" => Some( @@ -554,6 +615,11 @@ fn property_of(arg: &Arg) -> serde_json::Value { exist makes this run store its own graph rather than the deployed one.", ), "full_refresh" => Some("Rebuild incremental models from scratch instead of appending."), + "defer" => Some( + "Resolve a `ref()` this run does not build to the relation the last successful \ + run of this warehouse and target published, instead of to the schema this run \ + writes into.", + ), "model" => Some( "The model to preview, by name — `stg_orders`, or `my_package.stg_orders` when \ two packages share a name. Any dbt selector resolving to ONE node works.", @@ -687,8 +753,15 @@ full_refresh: true }; let (build, build_args) = of("build"); - assert_eq!(build_args, ["exclude", "full_refresh", "select", "vars"]); + assert_eq!( + build_args, + ["defer", "exclude", "full_refresh", "select", "vars"] + ); assert_eq!(build["properties"]["full_refresh"]["type"], "boolean"); + // `defer` is a per-run toggle rather than a descriptor-only setting: the + // run that publishes an environment's state and the run that defers to + // it are two invocations of ONE script. + assert_eq!(build["properties"]["defer"]["type"], "boolean"); // Defaults come from the descriptor, so an untouched run reproduces it. assert_eq!( build["properties"]["select"]["default"], diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 843a83a310..30c6047255 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -29,10 +29,10 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, the warehouse named as the - /// workspace configures it. The scheme names the producer, the path stays - /// the relation — see `windmill_types::AssetKind::Dbt`. + /// A warehouse relation, `dbt:////`, the warehouse + /// named as the workspace configures it. The scheme names the namespace dbt + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } @@ -288,11 +288,13 @@ pub struct RetrySpec { } // `// materialize [manual] [append] [key=] [history] [track=]` -// — declares that this script produces a *managed* materialization of `` -// (a `ducklake://` table). By default the runtime generates the write DDL around +// — declares that this script produces ``. A `ducklake://` table is +// materialized *managed* by default: the runtime generates the write DDL around // the script's single trailing `SELECT` and owns idempotency, partition-state // and snapshot capture. `manual` is the escape hatch: the script writes its own -// DDL and the runtime only records state (track-only). The reconciliation +// DDL and the runtime only records state (track-only) — and it is the only mode a +// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy +// enforces that; see docs/dbt-runtime.md). The reconciliation // strategy options apply to managed mode: none → DELETE-by-partition + INSERT // (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). @@ -804,6 +806,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String { ) } +/// Whether a `dbt://` path names a whole relation, `//`. +/// +/// Every producer spells one that way — the manifest ingest derives it from +/// `relation_name`, a `// materialize` target is checked against it — so anything +/// else can be produced by nothing and read by nothing. Both sides of the deploy +/// ask here rather than counting segments themselves: a subscription and a write +/// that disagreed on the shape would refuse and accept the same string. +pub fn is_full_relation_path(path: &str) -> bool { + let mut segments = path.split('/'); + segments.clone().count() == 3 && !segments.any(str::is_empty) +} + /// A doubled delimiter inside a quoted identifier is that delimiter, literally — /// the same rule the worker's `split_relation` applies to `relation_name`. Both /// have to decode it or one spelling of a table becomes two graph nodes: the dbt @@ -1740,10 +1754,7 @@ mod pipeline_annotation_tests { // just stop being the same node and the cross-boundary cascade never fires. #[test] fn table_paths_from_every_spelling_canonicalize_to_one_key() { - let canonical = Some(( - AssetKind::Dbt, - Cow::Owned("main/analytics/orders".into()), - )); + let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into()))); for spelling in [ // Hand-written annotation. "dbt://main/analytics/orders", @@ -1765,6 +1776,17 @@ mod pipeline_annotation_tests { } } + /// The shape both halves of the deploy check against: a subscription and a + /// write that disagreed on it would refuse and accept the same string. + #[test] + fn a_whole_relation_is_three_non_empty_segments() { + assert!(is_full_relation_path("main/analytics/orders")); + assert!(is_full_relation_path("main/archive.sales/orders")); + for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] { + assert!(!is_full_relation_path(partial), "{partial} is not a relation"); + } + } + // A relation that overrode its database carries `.` in // one segment, and each half can be quoted independently. Stripping only // the outer pair leaves a key the manifest ingest never produces, so the @@ -1791,10 +1813,7 @@ mod pipeline_annotation_tests { // database qualifier. assert_eq!( parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), - Some(( - AssetKind::Dbt, - Cow::Owned("main/sales.v2/orders".into()) - )) + Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into()))) ); } @@ -1813,14 +1832,8 @@ mod pipeline_annotation_tests { "dbt://main/analytics/\"order\"\"s\"", "main/analytics/order\"s", ), - ( - "dbt://main/`da``ta`/`orders`", - "main/da`ta/orders", - ), - ( - "dbt://main/[my]]schema]/[orders]", - "main/my]schema/orders", - ), + ("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"), + ("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"), // And in one half of a database-qualified segment. ( "dbt://main/\"arch\"\"ive\".\"sales\"/orders", @@ -1839,8 +1852,14 @@ mod pipeline_annotation_tests { // apart. A lone delimiter treated as opening a quote would be dropped — // `sa"les` filed as `sales` — and the two derivations would split. for (decoded, spelled) in [ - ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), - ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/sa\"les/orders", + "dbt://main/\"sa\"\"les\"/orders", + ), + ( + "dbt://main/analytics/order\"s", + "dbt://main/analytics/\"order\"\"s\"", + ), ( "dbt://main/arch\"ive.sales/orders", "dbt://main/\"arch\"\"ive\".\"sales\"/orders", diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 61f8a66f85..67758cef32 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -70,11 +70,15 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) +dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) +dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) + FK: (workspace_id) -> workspace(id) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 09720b2203..4b86013e7d 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -6,6 +6,7 @@ use axum::{ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::Row; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use windmill_common::{ assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind}, db::UserDB, @@ -13,7 +14,9 @@ use windmill_common::{ utils::escape_ilike_pattern, }; -use windmill_api_auth::{build_scope_path_predicate, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_filter, build_scope_path_predicate, ApiAuthed, ScopePathFilter, +}; // Partition-range backfill preview. The logic (producer resolution, range // enumeration, status join) is enterprise: the `private` build compiles the @@ -33,6 +36,7 @@ pub fn workspaced_service() -> Router { .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) .route("/graph", get(asset_graph)) + .route("/column_lineage", get(dbt_column_lineage)) .route("/pipelines", get(list_pipeline_folders)) .route("/partitions", get(list_partitions)) .route("/partitions_in_range", get(list_partitions_in_range)) @@ -663,10 +667,21 @@ struct DbtAssetProvenance { description: Option, #[serde(skip_serializing_if = "Vec::is_empty")] data_tests: Vec, - /// Declared column metadata (name -> description). NOT column lineage — - /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14). + /// Declared column metadata (name -> description): what `manifest.json` + /// carries, which is only the columns an author wrote down. #[serde(skip_serializing_if = "Option::is_none")] columns: Option, + /// Every column of the relation, typed and in order — + /// `[{"name": …, "type": …}]` — from the engine's static analysis. Present + /// only for a project that opted into it. + /// + /// Gated exactly like `columns` and the model's SQL: a full column list is + /// the shape of what the author WROTE, one level finer than the `ref()` + /// graph, which is ungated only because it draws relations the caller + /// already sees in `asset`. Widening that boundary has to be a decision, not + /// a consequence of a project turning the analysis pass on. + #[serde(skip_serializing_if = "Option::is_none")] + column_schema: Option, /// A source's declared freshness policy, for the staleness chip. #[serde(skip_serializing_if = "Option::is_none")] freshness: Option, @@ -946,6 +961,442 @@ struct DbtLineageEdge { to_asset_path: String, } +/// One column-to-column edge, in the same terms: the two relations and the two +/// columns, never dbt's node ids. +#[derive(Serialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct DbtColumnLineageEdge { + from_asset_path: String, + from_column: String, + to_asset_path: String, + to_column: String, + /// dbt's own word for how the value travelled: `copy` (passthrough), `mod` + /// (transformed), `scan` (read to produce the ROW rather than the value — a + /// join key, a predicate, a `group by`). Sent verbatim, including a kind + /// this engine version invented, because the renderer decides what a kind + /// means and the set is the engine's. + kind: String, +} + +/// The dbt relations a view is tracing, and which stored graph to read them +/// from. +/// +/// Several relations, answered as one union, because ONE selection reaches +/// several: a script's output column can be derived from columns of several dbt +/// models, and a model's columns can be consumed by scripts that feed others. +/// Asking per relation instead is a request per boundary plus the bookkeeping to +/// stitch the answers together and decide which of them is still current — which +/// is a cache, and is what taking them together exists to not need. +pub struct ColumnLineageQuery { + /// The `dbt://` relations whose lineage to return. + pub asset_paths: Vec, + /// The deployed version a view is drawing, when it is drawing one — the dbt + /// editor, which shows a single project as of a single deploy. + /// + /// A version-pinned answer is that version's project ALONE, the same as a + /// job-pinned one: the pin exists so the trace describes the stored graph on + /// screen, and another project's live graph is not part of it. Only the + /// unpinned answer crosses projects. + /// + /// A run's or an editor buffer's graph is NOT reachable from here: it pins + /// to a job, and that costs the job-read gate. + pub dbt_script_hash: Option, +} + +impl ColumnLineageQuery { + /// Built from the raw pairs rather than deserialized as a struct, because + /// `asset_path` REPEATS and `serde_urlencoded` — what `Query` deserializes + /// with — reads no sequence from a repeated key. A GET rather than a POST + /// body carrying the list: the method decides a scoped token's action, so a + /// POST would ask `assets:write` for a read and refuse a read-only token + /// outright. + pub fn from_query_pairs(pairs: Vec<(String, String)>) -> windmill_common::error::Result { + let mut asset_paths: Vec = Vec::new(); + let mut dbt_script_hash = None; + for (key, value) in pairs { + match key.as_str() { + "asset_path" => asset_paths.push(value), + // Hex, like every other script-hash parameter, so a page can + // pass `job.script_hash` verbatim. + "dbt_script_hash" => { + dbt_script_hash = + Some(serde_json::from_value(Value::String(value)).map_err(|_| { + windmill_common::error::Error::BadRequest( + "dbt_script_hash is not a script hash".to_string(), + ) + })?) + } + _ => {} + } + } + // REFUSED, not answered empty. A caller that named no relation — or + // misspelled the parameter — asked for something, and an empty component + // is what a relation with no lineage returns, so answering that way says + // "this has none" for a question that was never asked. + if asset_paths.is_empty() { + return Err(windmill_common::error::Error::BadRequest( + "at least one asset_path is required".to_string(), + )); + } + Ok(ColumnLineageQuery { asset_paths, dbt_script_hash }) + } +} + +#[derive(Serialize, Debug, PartialEq, Eq)] +pub struct ColumnLineageResponse { + /// Direct (`copy` / `mod`) column edges of the component the asked-for + /// relations' columns sit in, in the terms the canvas draws. Empty when no + /// project involved asked for the analysis pass, which is the ordinary case. + edges: Vec, + /// The component reaches further than what is here: `edges` holds the part + /// nearest the asked-for relations. Said rather than silently cut, because a + /// trace that stops short is otherwise indistinguishable from one that ends. + truncated: bool, +} + +/// How many edges one trace may carry back. The renderer draws a box per column, +/// so a component past this is unreadable however it is served — a synthetic +/// 3000-model project whose models share a column returns 58k direct edges and +/// 7.3MB. Applied over the walk, which is the LAST filter, so what survives is +/// the part nearest the selection rather than an arbitrary slice of it. +const MAX_TRACE_EDGES: usize = 5_000; + +/// How many times one trace may discover a project it has not read yet. Each +/// round costs a gate and a fetch, and a component crossing this many projects +/// has already outgrown what the canvas can show; stopping says what the edge +/// bound says. +const MAX_OWNER_ROUNDS: usize = 8; + +/// How many edges one trace may HOLD while walking, as opposed to answer with. +/// A project's edges arrive whole — the walk decides what is in the component, +/// so a `LIMIT` on the fetch would cut an arbitrary set that need not even +/// contain the asked-for relation — and a project is bounded at ingest by +/// `MAX_COLUMN_EDGES`, which is 200k. Rounds are what this bounds: reading a +/// second project's worth on top of an already outsized first one buys nothing, +/// since the walk is going to stop at `MAX_TRACE_EDGES` regardless. +const MAX_HELD_EDGES: usize = 100_000; + +/// A stored project graph: a deployed version, or one job's snapshot of it. +/// `script_hash` is NULL for an editor buffer's parse, which names no version. +type ProjectVersion = (String, Option, uuid::Uuid); + +async fn dbt_column_lineage( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(pairs): Query>, +) -> JsonResult { + // `None`: pinning to one run is job-scoped and this endpoint is authorized + // as `assets:read`. See `dbt_column_lineage_for`. + let q = ColumnLineageQuery::from_query_pairs(pairs)?; + dbt_column_lineage_for(&authed, &w_id, user_db, q, None).await +} + +/// The column-level lineage the asked-for relations sit in, optionally as one +/// run saw it. +/// +/// AUTHORIZES NOTHING BY ITSELF, on the same contract as `asset_graph_for`: +/// `assets:read` always, and the job-read gate for `Some(pinned)`, whose path +/// and hash are then taken from that job's row rather than from the caller. +/// +/// A column trace is transitive and a relation is not owned by one project, so +/// the answer grows a project at a time: resolve who owns the relations reached +/// so far, read their edges, walk, and repeat for the relations that walk newly +/// reached. Every round re-applies the caller's gate to the projects it +/// discovers — a relation being reachable from a project the caller may read +/// says nothing about the project on the other side of it. +pub async fn dbt_column_lineage_for( + authed: &ApiAuthed, + w_id: &str, + user_db: UserDB, + q: ColumnLineageQuery, + pinned: Option, +) -> JsonResult { + // A column-level view is the shape of what the author WROTE, so it takes the + // model's own gate rather than the relation's. + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(authed, "scripts", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + let (pinned_path, script_hash) = match pinned.as_ref() { + // The job's own version, so a pin cannot name one project's run while + // claiming another's version — including when it names NONE, which is + // the editor buffer. + Some(p) => (Some(p.script_path.as_str()), p.script_hash), + None => (None, q.dbt_script_hash.map(|h| h.0)), + }; + let pinned_job_id = pinned.as_ref().map(|p| p.job_id); + + let seeds: BTreeSet = q.asset_paths.into_iter().collect(); + let mut tx = user_db.begin(authed).await?; + + // Relations whose owners have been asked for, project graphs already read, + // and the edges they yielded. These are what end the loop: a round asks only + // about relations not asked about before and reads only projects not read + // before, so it stops as soon as one of the two runs out. + let mut asked: HashSet = HashSet::new(); + let mut read: HashSet = HashSet::new(); + let mut edges: Vec = Vec::new(); + let mut answer: Vec = Vec::new(); + let mut pending: Vec = seeds.iter().cloned().collect(); + let mut truncated = false; + + for _ in 0..MAX_OWNER_ROUNDS { + if pending.is_empty() { + break; + } + // Which project version owns each of these relations, under this + // caller's access. Usually one row per relation; a relation a second + // project declares as a source has two, and each answers for its own + // lineage. + let owners = sqlx::query!( + r#"SELECT DISTINCT n.script_path AS "script_path!", n.script_hash, n.job_id AS "job_id!" + FROM dbt_node n + WHERE n.workspace_id = $1 AND n.asset_path = ANY($2) + -- The run's snapshot, or the deployed graph when that job stored + -- none -- a build pins only if it wrote one. + AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $5) + THEN $5::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END + -- The gate, re-decided for every project the walk reaches. That + -- is what resolving owners in a loop is for: being entitled to + -- one project is not being entitled to the one that declares a + -- relation it hands over. + AND ( $6 + OR n.script_path = ANY($7) + OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx + WHERE n.script_path = pfx + OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) ) + AND CASE + -- Pinned: which version comes from a job this caller was + -- already granted, so `script` does not decide THAT -- but + -- it still decides whether the project may be read, the + -- same second gate `script_visible` is on the graph. Being + -- entitled to a run is not being entitled to the SQL + -- behind it, and column lineage is that SQL's shape. A + -- version-less row is exempt because it is an editor + -- buffer, which has no `script` row to ask and reaches + -- this only through the parse job that wrote it. + -- + -- One project answers, so a pinned trace never crosses + -- into another: neither does the graph it annotates. + WHEN $4::text IS NOT NULL + THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint + AND ($3::bigint IS NULL OR EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3)) + -- A named version: the deployed one an editor is drawing. + -- `script` is read under RLS, so this is the visibility + -- check as well as the existence one. A hash names one + -- script row, so this arm answers for one project too — + -- and deliberately: a pin says which stored graph is on + -- screen, and another project's live graph is not it. + WHEN $3::bigint IS NOT NULL + THEN n.script_hash = $3 AND EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3) + -- Otherwise the version deployed now: an older one's rows + -- outlive it in `dbt_node` until the sweep, and describe a + -- project that is no longer what runs. `language` narrows + -- it the way the graph's own resolution does, so a path + -- that has since become a script of another kind draws and + -- explains the same version rather than disagreeing. Read + -- under RLS, so a project the caller cannot see resolves + -- to NULL and matches nothing. + ELSE n.script_hash = ( + SELECT sc.hash FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.language = 'dbt' + AND sc.deleted = false AND sc.archived = false + ORDER BY sc.created_at DESC LIMIT 1) + END"#, + w_id, + &pending[..], + script_hash, + pinned_path, + pinned_job_id, + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + asked.extend(pending.drain(..)); + + let fresh: Vec = owners + .into_iter() + .map(|o| (o.script_path, o.script_hash, o.job_id)) + .filter(|k| read.insert(k.clone())) + .collect(); + if fresh.is_empty() { + break; + } + let fresh_paths: Vec = fresh.iter().map(|k| k.0.clone()).collect(); + let fresh_hashes: Vec> = fresh.iter().map(|k| k.1).collect(); + let fresh_jobs: Vec = fresh.iter().map(|k| k.2).collect(); + + // DIRECT kinds only. `scan` -- the column was read to produce the ROW, + // not the value -- reaches every output column of its model, so it is + // most of a project's stored lineage and none of what a trace draws. + // It stays in the table for a later view to ask for. + let rows = sqlx::query!( + r#"SELECT p.asset_path AS "from_path!", e.parent_column AS "from_column!", + c.asset_path AS "to_path!", e.child_column AS "to_column!", + e.lineage_kind AS "kind!" + FROM unnest($2::text[], $3::bigint[], $4::uuid[]) + AS o(script_path, script_hash, job_id) + JOIN dbt_column_edge e ON e.workspace_id = $1 + AND e.script_path = o.script_path + AND e.job_id = o.job_id + -- `=` still, with the NULL-to-NULL case + -- spelled out and gated on the pin: a + -- version-less row's hash is NULL on both + -- sides, which `=` never matches, but + -- `IS NOT DISTINCT FROM` would cost the + -- equality its index bound everywhere else. + AND (e.script_hash = o.script_hash + OR ($5::text IS NOT NULL + AND o.script_hash IS NULL + AND e.script_hash IS NULL)) + JOIN dbt_node p ON p.workspace_id = e.workspace_id + AND p.script_path = e.script_path + AND p.script_hash IS NOT DISTINCT FROM e.script_hash + AND p.job_id = e.job_id + AND p.unique_id = e.parent_unique_id + JOIN dbt_node c ON c.workspace_id = e.workspace_id + AND c.script_path = e.script_path + AND c.script_hash IS NOT DISTINCT FROM e.script_hash + AND c.job_id = e.job_id + AND c.unique_id = e.child_unique_id + WHERE e.lineage_kind IN ('copy', 'mod') + AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL"#, + w_id, + &fresh_paths[..], + &fresh_hashes[..] as &[Option], + &fresh_jobs[..], + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + edges.extend(rows.into_iter().map(|r| DbtColumnLineageEdge { + from_asset_path: r.from_path, + from_column: r.from_column, + to_asset_path: r.to_path, + to_column: r.to_column, + kind: r.kind, + })); + // Two projects can describe one relation, so the same edge can arrive + // twice. Sorted as well as deduplicated: the walk reads the incidence + // lists in this order, so the answer does not depend on which round a + // project was discovered in. + edges.sort(); + edges.dedup(); + + let walked = component(&edges, &seeds); + answer = walked.edges; + truncated = walked.truncated; + if truncated { + break; + } + // The relations the walk newly reached. Their owners are the next round's + // question: this project declares them, and so may another. + pending = answer + .iter() + .flat_map(|e| [&e.from_asset_path, &e.to_asset_path]) + .filter(|p| !asked.contains(*p)) + .cloned() + .collect::>() + .into_iter() + .collect(); + // Stop discovering projects once the held set is outsized. The tail + // below reports what that leaves unresolved, and reports nothing when + // the walk had already reached everything. + if edges.len() >= MAX_HELD_EDGES { + break; + } + } + tx.commit().await?; + // Out of rounds with relations still unresolved: more of the component + // exists, which is what hitting the edge bound also means. + Ok(Json(ColumnLineageResponse { + truncated: truncated || !pending.is_empty(), + edges: answer, + })) +} + +struct WalkedComponent { + edges: Vec, + truncated: bool, +} + +/// The edges of the connected component the asked-for relations sit in, nearest +/// first and at most `MAX_TRACE_EDGES` of them. +/// +/// The canvas lays out the component of the selected relation's columns, so a +/// project's other model families are edges nothing it draws can reach. Walked +/// here rather than in SQL: a recursive CTE has no index to walk, so it rescans +/// the whole edge set once per level — measured at 1.24s against 59ms for the +/// query alone on a 3000-model project, for a walk that is microseconds over a +/// map. Columns are keyed by relation, not by project, which is how the canvas +/// keys them too: two projects describing one relation draw one node. +/// +/// Breadth-first, so the bound cuts the far end of the trace rather than an +/// arbitrary part of it. +fn component(edges: &[DbtColumnLineageEdge], seeds: &BTreeSet) -> WalkedComponent { + let mut incident: HashMap<(&str, &str), Vec> = HashMap::new(); + for (i, e) in edges.iter().enumerate() { + incident + .entry((&e.from_asset_path, &e.from_column)) + .or_default() + .push(i); + incident + .entry((&e.to_asset_path, &e.to_column)) + .or_default() + .push(i); + } + let mut start: Vec<(&str, &str)> = incident + .keys() + .filter(|(path, _)| seeds.contains(*path)) + .copied() + .collect(); + start.sort(); + let mut seen_node: HashSet<(&str, &str)> = start.iter().copied().collect(); + let mut queue: VecDeque<(&str, &str)> = start.into(); + let mut taken = vec![false; edges.len()]; + let mut kept: Vec = Vec::new(); + let mut truncated = false; + 'walk: while let Some(node) = queue.pop_front() { + for &i in incident.get(&node).map(Vec::as_slice).unwrap_or_default() { + if std::mem::replace(&mut taken[i], true) { + continue; + } + if kept.len() == MAX_TRACE_EDGES { + truncated = true; + break 'walk; + } + kept.push(i); + let e = &edges[i]; + let ends = [ + (e.from_asset_path.as_str(), e.from_column.as_str()), + (e.to_asset_path.as_str(), e.to_column.as_str()), + ]; + for end in ends { + if seen_node.insert(end) { + queue.push_back(end); + } + } + } + } + // Back into edge order, so a response does not carry the walk's shape. + kept.sort_unstable(); + WalkedComponent { edges: kept.into_iter().map(|i| edges[i].clone()).collect(), truncated } +} + async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, @@ -1323,7 +1774,7 @@ pub async fn asset_graph_for( n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path, n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description, n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node, - n.columns, n.freshness, + n.columns, n.column_schema, n.freshness, n.raw_code, n.original_file_path, -- Whether the caller may read the project this row describes. -- The query deliberately reaches outside the requested folder @@ -1379,6 +1830,10 @@ pub async fn asset_graph_for( // `ref()` lineage between two models, resolved to the relations they // produce. Joined to `dbt_node` on both key columns because a dbt // `unique_id` is only unique within its project. + // + // Column lineage is NOT here. It is stored per relation and per column, and + // this response is folder-wide and polled by a run page, so it carries only + // what the canvas draws for every node at once. let dbt_edge_rows = sqlx::query!( r#"WITH live AS ( SELECT * FROM ( @@ -1604,6 +2059,7 @@ pub async fn asset_graph_for( description: r.description.clone().filter(|_| source_allowed), data_tests: vec![], columns: r.columns.clone().filter(|_| source_allowed), + column_schema: r.column_schema.clone().filter(|_| source_allowed), freshness: r.freshness.clone().filter(|_| source_allowed), }; // One relation can carry rows from several projects — typically a model diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs index f2bb98a46c..75299f4d9f 100644 --- a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -8,7 +8,9 @@ //! whole dbt half, and every later fix in this area re-touched one of the two. use sqlx::{Pool, Postgres}; -use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun}; +use windmill_api_assets::{ + asset_graph_for, dbt_column_lineage_for, ColumnLineageQuery, GraphQuery, PinnedRun, +}; use windmill_api_auth::ApiAuthed; use windmill_common::db::UserDB; @@ -84,6 +86,34 @@ async fn seed(db: &Pool, job: uuid::Uuid) { .execute(db) .await .unwrap(); + // The relation it reads, and the column edge between them. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders', + 'u/a/wh/analytics/raw_orders', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', + 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); // A test node, for the arguments it carries: `accepted_values` spells out a // column's domain. sqlx::query!( @@ -365,7 +395,25 @@ async fn seed_editor_graph(db: &Pool, job: uuid::Uuid) { r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, raw_code, tags) VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft', - 'u/a/wh/analytics/draft', 'select 3', '{}')"#, + 'u/a/wh/analytics/draft', 'select 3', '{}'), + ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src', + 'u/a/wh/analytics/draft_src', 'select 4', '{}')"#, + WS, + PATH, + job + ) + .execute(db) + .await + .unwrap(); + // A version-less row's `script_hash` is NULL on both sides of every join and + // every visibility check, and `= NULL` is never true — so the column edges + // need the same NULL arm the node query has, or a buffer parse renders its + // columns and none of their lineage. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", WS, PATH, job @@ -451,3 +499,444 @@ async fn an_editor_graph_renders_only_through_its_own_job(db: Pool) { "nor of a run of the deployed version: {deployed_run}" ); } + +async fn column_lineage_q( + db: &Pool, + authed: &ApiAuthed, + pairs: Vec<(String, String)>, + pinned: Option, +) -> serde_json::Value { + let res = dbt_column_lineage_for( + authed, + WS, + UserDB::new(db.clone()), + ColumnLineageQuery::from_query_pairs(pairs).unwrap(), + pinned, + ) + .await + .unwrap(); + serde_json::to_value(&res.0).unwrap() +} + +async fn column_lineage( + db: &Pool, + authed: &ApiAuthed, + asset_paths: &[&str], + pinned: Option, +) -> serde_json::Value { + let pairs = asset_paths + .iter() + .map(|p| ("asset_path".to_string(), p.to_string())) + .collect(); + column_lineage_q(db, authed, pairs, pinned).await +} + +/// The buffer parse's own lineage, which is the case the versionless rows exist +/// for. Its `script_hash` is NULL on both sides of every join and every +/// visibility check, and `= NULL` is never true — so the versionless arm has to +/// be written for it, or a parse renders its columns and none of their lineage. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_editor_buffers_column_lineage_answers_through_its_job(db: Pool) { + let parse = uuid::Uuid::from_u128(9); + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_editor_graph(&db, parse).await; + let admin = ApiAuthed { is_admin: true, ..outsider() }; + + let pinned = PinnedRun { job_id: parse, script_path: PATH.to_string(), script_hash: None }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], Some(pinned)).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/draft_src", + "from_column": "raw", + "to_asset_path": "u/a/wh/analytics/draft", + "to_column": "clean", + "kind": "mod", + }]), + ); + // Unpinned, the same relation resolves through the deployed version, which + // never heard of the buffer's models. + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], None).await["edges"], + serde_json::json!([]), + "a buffer's lineage is reachable only through the job that parsed it" + ); +} + +/// Being entitled to a RUN is not being entitled to the SQL behind it, and +/// column lineage is that SQL's shape. The pinned graph draws the relations for +/// a share-link viewer and redacts what the author wrote; the lineage is the +/// second, and resolving the version from the job must not be mistaken for +/// deciding that too. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_pinned_run_does_not_hand_over_the_projects_column_lineage(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + + assert_eq!( + column_lineage( + &db, + &outsider(), + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([]), + "the run renders for them, its column-level shape does not" + ); + assert_eq!( + column_lineage( + &db, + &ApiAuthed { is_admin: true, ..outsider() }, + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "while a reader of the project gets it" + ); +} + +/// The deployed version of the same two models, plus a `scan` edge beside the +/// direct one: `seed`'s rows are a run's snapshot, and the unpinned answer is +/// the version's own graph. +async fn seed_deployed_orders(db: &Pool) { + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders', + 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders', + 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); +} + +/// A column-level view is the shape of what the author WROTE, so it takes the +/// script's own gate — the same one that keeps `raw_code` behind access to the +/// project. `scan` says the column was read to produce the ROW rather than the +/// value, so it reaches every output column of its model and is never served. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_takes_the_scripts_gate_and_only_the_direct_kinds(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the direct edge, and not the `scan` one beside it" + ); + assert_eq!( + column_lineage(&db, &outsider(), &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([]), + "and nothing at all for a caller who cannot read the project" + ); +} + +/// One project routinely holds model families that share no column, and the +/// canvas lays out the connected component of the selected relation's columns. +/// Answering with the project's other components sends edges nothing can draw. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_stops_at_the_selected_relations_component(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + // A second family in the same project version, reaching neither of the two + // relations `seed` wired together. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock', + 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily', + 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], Some(pinned())).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the orders family, and not the stock one beside it in the same project" + ); + assert_eq!( + column_lineage( + &db, + &admin, + &["u/a/wh/analytics/stock_daily"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + "and the other way round — reached from the child end, which is upstream" + ); +} + +/// A deployed dbt project in `folder`, declaring `parent`'s relation as a source +/// and deriving `child` from it. `orders → mart → secret_out` is three projects +/// chained through two shared relations. +async fn seed_neighbour_project( + db: &Pool, + folder: &str, + hash: i64, + parent: (&str, &str), + child: (&str, &str), +) { + let path = format!("f/{folder}/proj"); + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) + VALUES ($1, $2, $2, '{}', '{}')", + WS, + folder + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + hash, + path, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4, + 'source', $4, $5, '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6, + 'model', $6, $7, '{}')", + WS, + path, + hash, + parent.0, + parent.1, + child.0, + child.1, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + WS, + path, + hash, + parent.0, + child.0, + ) + .execute(db) + .await + .unwrap(); +} + +/// A trace crosses out of the project it started in, and the gate crosses with +/// it. +/// +/// A relation one project produces is another's source, so the component reaches +/// edges the first project's owner set never named — that is what resolving +/// owners to a fixpoint is for. The other half is that the caller's access has +/// to be re-decided for each project discovered on the way: reaching a relation +/// says nothing about who may read the project on the far side of it. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_crosses_projects_only_where_the_caller_may_read_them(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + seed_neighbour_project( + &db, + "mid", + 43, + ("orders", "u/a/wh/analytics/orders"), + ("mart", "u/a/wh/analytics/mart"), + ) + .await; + seed_neighbour_project( + &db, + "secret", + 44, + ("mart", "u/a/wh/analytics/mart"), + ("secret_out", "u/a/wh/analytics/secret_out"), + ) + .await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let reached = |body: &serde_json::Value| { + body["edges"] + .as_array() + .unwrap() + .iter() + .map(|e| e["to_asset_path"].as_str().unwrap().to_string()) + .collect::>() + }; + + let all = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&all), + [ + "u/a/wh/analytics/mart", + "u/a/wh/analytics/orders", + "u/a/wh/analytics/secret_out" + ] + .map(String::from) + .into(), + "two projects out from the one asked about, not one: {all}" + ); + + // Granted the first two folders and not the third. The edges of the project + // they may read are theirs; the one beyond it is not, even though the + // relation joining them is in the answer. + let partial = ApiAuthed { + folders: vec![ + ("private".to_string(), false, false), + ("mid".to_string(), false, false), + ], + ..outsider() + }; + let some = column_lineage(&db, &partial, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&some), + ["u/a/wh/analytics/mart", "u/a/wh/analytics/orders"] + .map(String::from) + .into(), + "the trace stops where the caller's access does: {some}" + ); + + // The other half of the same gate, and the half that is hand-written SQL + // rather than RLS: a token scoped to one folder reaches the projects in it + // and no others, whatever its grants say. + let scoped = ApiAuthed { + is_admin: true, + scopes: Some(vec!["scripts:read:f/private/*".to_string()]), + ..outsider() + }; + let scoped = column_lineage(&db, &scoped, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&scoped), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "and where its scope does: {scoped}" + ); + + // A version pin answers for that version's project alone — the dbt editor, + // which draws one project as of one deploy. Crossing into `mid` here would + // annotate that canvas with relations it does not draw. + let pinned_version = column_lineage_q( + &db, + &admin, + vec![ + ( + "asset_path".to_string(), + "u/a/wh/analytics/orders".to_string(), + ), + ("dbt_script_hash".to_string(), format!("{:016x}", HASH)), + ], + None, + ) + .await; + assert_eq!( + reached(&pinned_version), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "a version pin does not cross into the project beside it: {pinned_version}" + ); +} + +/// The bound on the answer, and that hitting it is said rather than silently +/// cut: a trace that stops reads exactly like one that ends. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_component_past_the_bound_is_cut_and_says_so(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // One direct edge per column pair, more of them than a trace may carry. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 6000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!(body["edges"].as_array().unwrap().len(), 5000); + assert_eq!(body["truncated"], serde_json::json!(true)); +} diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index c6f88dace5..07b8004d83 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -318,6 +318,19 @@ async fn add_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } @@ -528,6 +541,19 @@ async fn remove_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } } diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs new file mode 100644 index 0000000000..60f846ca1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response { + authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap() +} + +/// A `dbt://` relation is one graph node only while every side spells it the same +/// way, and three sides derive that spelling independently: the `// materialize` +/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and +/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer` +/// prove the predicate; only a deploy proves the handler feeds it the key the +/// table actually holds — so a canonicalization that drifted on one side would +/// pass those and split the node here. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + sqlx::query!( + r#"UPDATE workspace_settings + SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb + WHERE workspace_id = 'test-workspace'"# + ) + .execute(&db) + .await?; + + // Nothing generates warehouse DDL, so a managed target is refused rather than + // degraded into the track-only mode it would silently become. + let resp = deploy( + port, + "u/test-user/managed", + "// materialize dbt://main/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("must be `manual`")); + + // The warehouse segment is the identity a dbt model keys on; a name the + // workspace does not configure strands the write on an unreachable node. + let resp = deploy( + port, + "u/test-user/unknown_wh", + "// materialize manual dbt://nope/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("does not configure")); + + // Only the DuckDB executor runs `// data_test` probes, and it runs them around + // a managed write — so a declarer in another language would deploy green with + // its assertions silently never executed. + let resp = deploy( + port, + "u/test-user/tested", + "// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp + .text() + .await? + .contains("`// data_test` is not supported")); + + // Both halves are held to the same relation: every producer is a whole + // `//` under a configured warehouse, so a + // subscription to anything else names something nothing can ever write. + for (path, ref_, expected) in [ + ( + "u/test-user/partial_sub", + "dbt://main/analytics", + "not a whole warehouse relation", + ), + ( + "u/test-user/unknown_wh_sub", + "dbt://nope/analytics/orders", + "does not configure", + ), + ] { + let resp = deploy( + port, + path, + &format!("// on {ref_}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains(expected)); + } + + // Past `asset.path`'s column, where the manifest ingest drops the relation and + // no producer row can exist on either side — computed from the bound so it + // cannot drift under it. + let overlong = format!( + "main/analytics/{}", + "o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN) + ); + let resp = deploy( + port, + "u/test-user/overlong_sub", + &format!("// on dbt://{overlong}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("characters an asset path holds")); + + // Any language may declare the write — the DuckLake write engine is DuckDB's, + // this declaration is not — and the target is canonicalized on the way into + // `asset`, so a hand-written mixed-case spelling lands on the model's key. + let resp = deploy( + port, + "u/test-user/ingest", + "// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + // The create response, not `{:x}` over the stored i64: `ScriptHash` decodes + // hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash + // under 2^60 would 422 the rename below instead of reaching the refusal. + let ingest_hash = resp.text().await?; + let write = sqlx::query_scalar!( + "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \ + AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'" + ) + .fetch_one(&db) + .await?; + assert_eq!(write, "main/analytics/orders"); + + // That producer is native, so subscribing to what it writes is accepted — and + // the `// on` ref has to canonicalize identically, or the row it stores names + // a relation nothing produces. + let resp = deploy( + port, + "u/test-user/consumer", + "// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + let trigger_ref = sqlx::query_scalar!( + "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \ + AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'" + ) + .fetch_one(&db) + .await?; + assert_eq!(trigger_ref, "dbt://main/analytics/orders"); + + // With dbt as the only producer the same subscription can never be woken — a + // dbt run does not dispatch — so the deploy refuses it and names the project. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = deploy( + port, + "u/test-user/mart_consumer", + "// on dbt://main/analytics/MARTS\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // A rename is the other half of that: the producer's write still sits at the + // OLD path in the committed snapshot this deploy reads, while the same + // transaction removes it — so it must not count as the producer that would + // wake the subscription the rename adds. + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/ingest_renamed", + "parent_hash": ingest_hash, + "summary": "", + "description": "", + "content": "// on dbt://main/analytics/orders\nexport async function main() {}", + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/protection_rules.rs b/backend/windmill-api-integration-tests/tests/protection_rules.rs index 9469554999..e032acef7b 100644 --- a/backend/windmill-api-integration-tests/tests/protection_rules.rs +++ b/backend/windmill-api-integration-tests/tests/protection_rules.rs @@ -123,6 +123,72 @@ async fn test_protection_rules(db: Pool) -> anyhow::Result<()> { .await?; assert!(!resp.status().is_success(), "Non-admin should be blocked from flows: {}", resp.status()); + // ======================================== + // 4b. ...but a draft-only resource stays deletable: nothing is deployed at + // its path, so its DELETE is a draft discard rather than a deployment. + // ======================================== + + let draft_only_path = "u/test-user-2/draft_only_resource"; + let resp = authed( + client().post(format!("{base}/drafts/update/resource/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ "value": { + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + }})) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Non-admin should save a draft: {}", + resp.text().await? + ); + + let resp = authed( + client().delete(format!("{base}/resources/delete/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Draft-only delete should not be gated by the deploy rules: {}", + resp.text().await? + ); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'resource'::DRAFT_KIND", + ) + .bind(draft_only_path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + // The gate itself is still there for a DEPLOYED resource at the same path. + let resp = authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "Non-admin should still be blocked from creating a resource: {}", + resp.status() + ); + // ======================================== // 5. Admin bypasses protection rule // ======================================== diff --git a/backend/windmill-api-integration-tests/tests/schedules.rs b/backend/windmill-api-integration-tests/tests/schedules.rs index 89ba0ec2cc..881f11ff43 100644 --- a/backend/windmill-api-integration-tests/tests/schedules.rs +++ b/backend/windmill-api-integration-tests/tests/schedules.rs @@ -113,7 +113,9 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { "expected at least 2 schedules, got {}", list.len() ); - assert!(list.iter().any(|s| s["path"] == "u/test-user/test_schedule")); + assert!(list + .iter() + .any(|s| s["path"] == "u/test-user/test_schedule")); // --- list_with_jobs --- let resp = authed(client().get(format!("{base}/list_with_jobs"))) @@ -125,18 +127,14 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert!(!list.is_empty()); // --- update --- - let resp = authed(client().post(schedule_url( - port, - "update", - "u/test-user/test_schedule", - ))) - .json(&json!({ - "schedule": "0 0 */12 * * *", - "timezone": "Europe/Paris" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(schedule_url(port, "update", "u/test-user/test_schedule"))) + .json(&json!({ + "schedule": "0 0 */12 * * *", + "timezone": "Europe/Paris" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); // verify update @@ -204,14 +202,11 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(resp.status(), 200); // --- delete --- - let resp = authed(client().delete(schedule_url( - port, - "delete", - "u/test-user/another_schedule", - ))) - .send() - .await - .unwrap(); + let resp = + authed(client().delete(schedule_url(port, "delete", "u/test-user/another_schedule"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let resp = authed_get(port, "exists", "u/test-user/another_schedule").await; @@ -220,17 +215,108 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Global endpoints ===== // --- preview --- - let resp = authed(client().post(format!( - "http://localhost:{port}/api/schedules/preview" - ))) - .json(&json!({ - "schedule": "0 0 */6 * * *", - "timezone": "UTC" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(format!("http://localhost:{port}/api/schedules/preview"))) + .json(&json!({ + "schedule": "0 0 */6 * * *", + "timezone": "UTC" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "preview: {}", resp.text().await?); Ok(()) } + +/// A schedule with no `schedule` row is listed from the `draft` table, so its +/// DELETE drops that draft, then 404s once nothing is left at the path. A legacy +/// (`email IS NULL`) draft is owned by nobody and stays put. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_draft_only_schedule(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/draft_only_schedule"; + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/trigger_schedule/{path}" + ))) + .json(&json!({ "value": { + "path": path, + "schedule": "0 0 */6 * * *", + "timezone": "UTC", + "script_path": "u/test-user/never_deployed", + "is_flow": false, + }})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "save draft: {}", resp.text().await?); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/schedules/list?include_draft_only=true" + ))) + .send() + .await + .unwrap(); + let listed: Vec = resp.json().await?; + assert!( + listed + .iter() + .any(|s| s["path"] == path && s["draft_only"] == json!(true)), + "draft-only schedule should be listed: {listed:?}" + ); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "nothing left at the path"); + + // A legacy (email IS NULL) draft is owned by nobody and keeps the write gate + // on the drafts routes, so this one must not become a second door to it. + let legacy_path = "u/test-user/legacy_draft_only_schedule"; + sqlx::query( + "INSERT INTO draft (workspace_id, email, path, typ, value) \ + VALUES ('test-workspace', NULL, $1, 'trigger_schedule'::DRAFT_KIND, '{}'::json)", + ) + .bind(legacy_path) + .execute(&db) + .await?; + + let resp = authed(client().delete(schedule_url(port, "delete", legacy_path))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 404, + "legacy draft is not this route's to delete" + ); + + let legacy_remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(legacy_path) + .fetch_one(&db) + .await?; + assert_eq!(legacy_remaining, 1, "the legacy draft should survive"); + + Ok(()) +} diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 6ae902e984..1b6b49a136 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -31,8 +31,8 @@ use windmill_common::{ self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, }, user_drafts::{ - delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, - UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, fetch_draft_only_list_rows, + overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, }, utils::{ escape_ilike_pattern, not_found_if_none, paginate, Pagination, ScheduleType, StripPath, @@ -1331,6 +1331,18 @@ async fn delete_schedule( .flatten(); if exists.is_none() { + drop(tx); + if delete_draft_only_for_path( + &db, + &w_id, + UserDraftItemKind::TriggerSchedule, + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only schedule {} deleted", path)); + } return Err(windmill_common::error::Error::NotFound(format!( "Schedule {} not found", path diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index ccc5815aa7..97e3019835 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,8 +39,8 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_common::{ assets::{ @@ -1081,6 +1081,57 @@ fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { [(path.to_string(), hash_script(lock))] } +/// The `dbt://` relation both halves of a deploy have to agree on: a whole +/// `//`, under a warehouse this workspace configures. +/// +/// Every producer is held to exactly this — a `// materialize` target here, a +/// descriptor's `profile.warehouse` in the worker — so a subscription to anything +/// else names a relation nothing can ever write. No later deploy fixes that and +/// no dormant-edge warning reports it, since the warning fires on a dbt project's +/// ingest and no project can claim a relation under a warehouse that isn't there. +/// Asking here rather than at each site is what keeps the two from drifting into +/// refusing and accepting the same string. +async fn validate_dbt_relation( + db: &sqlx::Pool, + w_id: &str, + relation: &str, + what: &str, +) -> Result<()> { + if !windmill_parser::asset_parser::is_full_relation_path(relation) { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is not a whole warehouse relation \ + (`dbt:////`)." + ))); + } + // `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that + // outgrows it rather than failing the whole graph, so past the column no + // producer row can exist on either side — a write would be rejected by + // Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text + // that would take the subscription and keep it dormant for good. + let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN; + if relation.chars().count() > max { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is longer than the {max} characters an asset path \ + holds, so it cannot be recorded." + ))); + } + let warehouse = relation.split('/').next().unwrap_or_default(); + // Only the resolver's own "no such warehouse" is the annotation's fault. Its + // other failures — the query, and a setting entry with no `resource_path` — + // keep their own error: blaming the warehouse name for those misdescribes + // them, and flattening the malformed-setting one to a 400 hides a server + // fault behind a client one. + windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse) + .await + .map_err(|e| match e { + Error::NotFound(_) => Error::BadRequest(format!( + "{what} `dbt://{relation}` names a warehouse this workspace does not \ + configure: {e}" + )), + other => other, + }) +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1565,34 +1616,90 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// materialize` materializes a `ducklake:///` target from a - // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake - // target would otherwise deploy, register a producer in the asset graph, then - // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), - // and a non-DuckDB script never reaches the executor that records state. The - // managed-only checks (single trailing SELECT, no SQL args) come after — a - // `manual` script owns its DDL and skips them. + // `// materialize` names what this script produces. Two target kinds, and the + // runtime behind each is what constrains the annotation: + // • `ducklake:///
` — the DuckDB executor generates the write + // (or, in `manual` mode, records the state the script wrote itself), so + // the script has to be a DuckDB one and the target has to name a table. + // A non-DuckDB script never reaches that executor. + // • `dbt:////` — a warehouse relation. Nothing + // generates warehouse DDL, so the declaration is track-only (`manual`) + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. + // Any other kind would deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). + // The managed-only checks (single trailing SELECT, no SQL args) come after — + // a `manual` script owns its DDL and skips them. if let Some(m) = pipeline_annotations.materialize.as_ref() { - if ns.language != ScriptLang::DuckDb { - return Err(Error::BadRequest(format!( - "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ - wmll.ducklake helpers to materialize from other languages.", - ns.language.as_str() - ))); - } - if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + use windmill_parser::asset_parser::AssetKind as PAssetKind; + // The producer half of the rule the trigger loop below applies to `// on`: + // a dbt project's writes come from its manifest, and the graph ingest + // republishes this path's asset rows wholesale, so a declared one would be + // wiped by the very deploy that accepted it while its runs kept stamping + // the relation. + if ns.language == ScriptLang::Dbt { return Err(Error::BadRequest( - "`// materialize` only supports a DuckLake target \ - (`ducklake:///
`); other asset kinds aren't materializable." + "a dbt script cannot declare `// materialize`: what a project builds is read \ + from its manifest and published by the graph ingest, not annotated." .to_string(), )); } - if !m.target_path.contains('/') { - return Err(Error::BadRequest(format!( - "`// materialize` needs a table in the target: \ - `ducklake://{0}/
` (got `ducklake://{0}`).", - m.target_path - ))); + match m.target_kind { + PAssetKind::Ducklake => { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages, or declare a \ + warehouse relation with `// materialize manual dbt://…`.", + ns.language.as_str() + ))); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + } + PAssetKind::Dbt => { + // `// data_test` runs as verifier probes the DuckDB executor + // splices around a MANAGED write. Nothing generates a warehouse + // write, so nothing would run them — and unlike the DuckLake + // `manual` case, which at least fails loudly in that executor, a + // declarer in another language would deploy green with its + // data-quality assertions silently never executed. + if !pipeline_annotations.data_tests.is_empty() { + return Err(Error::BadRequest( + "`// data_test` is not supported with a `dbt://` target: the checks run \ + against a managed materialization, and a warehouse relation is \ + written by the script itself. Assert on the relation with a dbt \ + test in the project that reads it." + .to_string(), + )); + } + if !m.manual { + return Err(Error::BadRequest( + "`// materialize dbt://…` must be `manual`: Windmill generates no \ + warehouse DDL, so the script issues its own write and only the outcome \ + is recorded. Write \ + `// materialize manual dbt:////`." + .to_string(), + )); + } + validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target") + .await?; + } + _ => { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake (`ducklake:///
`) or \ + warehouse-relation (`dbt:////`) target; other asset \ + kinds aren't materializable." + .to_string(), + )); + } } if !m.manual { if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { @@ -2347,27 +2454,28 @@ async fn create_script_internal<'c>( // while its own finished runs still render from them. Clearing by path // would empty those run pages for good. if ns.language != ScriptLang::Dbt { - // The saved retry state does go: nothing regenerates it, it is keyed by - // path alone, and it carries one user's failed invocation and its - // arguments. No dbt version is live at this path any more to resume it. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, &ns.path).await?; + // The saved run and environment state do go: nothing regenerates them, + // both are keyed by path alone, and they carry one user's failed + // invocation with its arguments and the project's own manifest. No dbt + // version is live at this path any more to resume or defer to. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, &ns.path).await?; } if let Some(ref old) = p_path_opt { if old != &ns.path { clear_script_triggers(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; clear_static_asset_usage(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; - // The saved retry state travels rather than being cleared: nothing + // The saved state travels rather than being cleared: nothing // regenerates it, so dropping it would throw away a resumable - // failure for what is only a rename. Only while the destination is - // still dbt — a rename that also converts the language would - // otherwise reinstate at the new path the state the branch above - // just cleared, leaving one user's arguments and results under a - // path no dbt script occupies. + // failure and every deferral until the next full run, for what is + // only a rename. Only while the destination is still dbt — a rename + // that also converts the language would otherwise reinstate at the + // new path the state the branch above just cleared, leaving one + // user's arguments and results under a path no dbt script occupies. if ns.language == ScriptLang::Dbt { - windmill_common::dbt_manifest::move_dbt_run_state(&mut tx, &w_id, old, &ns.path) + windmill_common::dbt_manifest::move_dbt_script_state(&mut tx, &w_id, old, &ns.path) .await?; } else { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, old).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, old).await?; } } } @@ -2375,16 +2483,43 @@ async fn create_script_internal<'c>( let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { continue; }; - // A `dbt://` subscription can never fire: dbt is the only producer of a - // warehouse relation (`// materialize` takes DuckLake targets only) and a - // dbt run does not dispatch. Refusing beats persisting a row that draws a - // cascade arrow on the canvas and then never wakes anything. - if trigger_ref.starts_with("dbt://") { - return Err(Error::BadRequest(format!( - "`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \ - runs, and nothing else writes a warehouse relation. Declare the read without \ - `on` to keep the lineage edge, or schedule this script." - ))); + // A `dbt://` subscription fires only when a NON-dbt job materialized the + // relation: `// materialize manual dbt://…` declares such a write, while a + // dbt run records its models and does not dispatch. So refuse exactly the + // edge that cannot fire — one whose relation is already claimed by dbt and + // by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`, + // which takes the workspace pool: under RLS an unreadable native producer + // would refuse a live subscription). + if let Some(relation) = trigger_ref.strip_prefix("dbt://") { + // The subscriber side of the same rule: a dbt project is not woken by + // the asset cascade. Its graph ingest clears these rows for its own + // path, so accepting one here would deploy an edge the dependency job + // then silently removes. + if ns.language == ScriptLang::Dbt { + return Err(Error::BadRequest(format!( + "a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \ + and a project is run on its schedule, not woken by an asset cascade." + ))); + } + validate_dbt_relation(&db, &w_id, relation, "subscription target").await?; + // Both paths under a rename: the old one's committed write row is + // still there and this transaction is about to remove it. + let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) { + Some(old) => vec![ns.path.clone(), old.to_string()], + None => vec![ns.path.clone()], + }; + if let Some(dbt_owner) = + windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths) + .await? + { + return Err(Error::BadRequest(format!( + "`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \ + `{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \ + read without `on` to keep the lineage edge, or schedule this script. A \ + relation written by a `// materialize manual {trigger_ref}` script can be \ + subscribed to." + ))); + } } // Effective debounce for this edge: per-`// on debounce=` wins, // else the script-level `// debounce` default. Debounce only @@ -3578,7 +3713,11 @@ async fn archive_script_by_path( path, &w_id ) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as the by-hash routes are: + // committed on its own, a cleanup that then fails leaves dbt state at a path + // no live version occupies, for whatever is created there next to defer + // through. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; @@ -3586,9 +3725,10 @@ async fn archive_script_by_path( // The graph stays: the pinned read resolves versions through a CTE that // already skips archived rows, so it stops answering for current relations // either way, while deleting it would empty the Models panel of every - // completed run of the project. Retry state does go — nothing may resume a - // script that is no longer live. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + // completed run of the project. The saved run and environment state do go — + // nothing may resume a script that is no longer live, and nothing may defer + // through what it last built. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; // Pipeline event hygiene: an archived script must not be triggered by // anything. Wipe declared `// on ...` edges (asset-event subscribers // look these up). @@ -3673,7 +3813,7 @@ async fn archive_script_by_hash( clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; // The version's graph stays: its finished runs still render from it, and // the live-version CTE already skips archived rows. Deletion clears it. - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3736,7 +3876,12 @@ async fn delete_script_by_hash( ) .bind(&hash.0) .bind(&w_id) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as `archive_script_by_hash` + // already does. Committed on its own, it opens a window where the path has + // no live version and a concurrent deploy can take it — and the retirement + // guard below then finds that new script live, keeps the old project's dbt + // state, and leaves the replacement able to defer through its manifest. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; @@ -3749,7 +3894,7 @@ async fn delete_script_by_hash( windmill_common::dbt_manifest::clear_dbt_manifest_version(&mut tx, &w_id, &script.path, hash.0) .await?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3850,11 +3995,11 @@ async fn delete_script_by_path( // After the DELETE, never before: every dbt writer locks the `script` row // first, so taking a sidecar ahead of it deadlocks one of the pair. The - // VERSIONED graph needs no clear at all, cascading off `script`; the retry - // state does, being keyed by path alone and so inherited by whatever is - // created here next, and so do the editor's own graphs, whose NULL + // VERSIONED graph needs no clear at all, cascading off `script`; the saved + // run and environment state do, being keyed by path alone and so inherited + // by whatever is created here next, and so do the editor's own graphs, whose NULL // `script_hash` satisfies that foreign key without riding its cascade. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, path).await?; if !trash_scripts.is_empty() { @@ -4023,7 +4168,7 @@ async fn delete_scripts_bulk( // Same reason as the single-path delete, over every requested path rather // than the deleted ones: a path that had no script left can still hold state. for p in &request.paths { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, p).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, p).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, p).await?; } diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c9343840d2..93e0ad5ca8 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -6627,7 +6627,7 @@ async fn clone_scripts( } /// The parsed dbt graph a deployed script carries: its models, their SQL and -/// tests, and the `ref()` lineage between them. +/// tests, and the `ref()` and column-level lineage between them. /// /// Keyed on (workspace_id, script_path, script_hash), and the fork keeps every /// script's hash, so each row moves across as itself. @@ -6645,11 +6645,11 @@ async fn clone_dbt_graph( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at) + columns, column_schema, freshness, raw_code, original_file_path, ingested_at) SELECT $2, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at + columns, column_schema, freshness, raw_code, original_file_path, ingested_at FROM dbt_node WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", source_workspace_id, @@ -6669,6 +6669,24 @@ async fn clone_dbt_graph( ) .execute(&mut **tx) .await?; + // Column lineage travels with the rest of the graph, and it has to: the + // snapshot's digest covers it, so a fork missing these rows recomputes the + // digest the source stored, matches, and stores nothing — leaving the + // lineage gone until someone redeploys, which is the failure this whole + // function exists to prevent. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind, + ingested_at) + SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column, + child_unique_id, child_column, lineage_kind, ingested_at + FROM dbt_column_edge + WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + source_workspace_id, + target_workspace_id + ) + .execute(&mut **tx) + .await?; sqlx::query!( "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest, relation_root_at_last_ingest, ingested_at) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 517d4c7af2..0ce3b21086 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.804.0 + version: 1.805.0 title: Windmill API contact: @@ -8528,6 +8528,84 @@ paths: items: type: string + /w/{workspace}/resources/type/resource_counts: + get: + summary: count the workspace's resources per resource type + operationId: listResourceCountsByType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: resource count per resource_type + content: + application/json: + schema: + type: array + items: + type: object + properties: + resource_type: + type: string + count: + type: integer + required: + - resource_type + - count + + /w/{workspace}/resources/type/hub/info: + get: + summary: list what the hub knows about its resource types + operationId: listHubResourceTypeInfo + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: each hub resource type with its integration and pick count, empty if the hub answers neither read + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + app: + description: the integration the resource type belongs to, which is not always its own name + type: string + picks: + type: integer + required: + - name + - app + - picks + + /w/{workspace}/resources/type/hub/pick/{name}: + post: + summary: record a hub resource type pick + operationId: pickHubResourceType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Name" + responses: + "200": + description: whether the hub recorded the pick + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: + - success + /w/{workspace}/npm_proxy/config: get: summary: get npm proxy configuration @@ -8777,6 +8855,9 @@ paths: properties: name: type: string + picks: + description: how often the integration has been picked, absent on a hub that does not count picks + type: integer required: - name @@ -24364,6 +24445,68 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/assets/column_lineage: + get: + summary: Column-level lineage of a set of dbt relations + description: > + The direct (`copy` / `mod`) column-to-column lineage the given relations' + columns sit in — the connected component around them, from the engine's + static analysis. Not their own edges, which would stop one hop out since + a column trace walks transitively, and not a whole project's, which + carries model families the selection cannot reach. + + Several relations, answered as one union, because one selection reaches + several: a script's output column can derive from columns of several dbt + models. Unpinned, the component crosses projects — a relation one project + produces is another's source — and the caller's access is decided again + for every project it reaches, so a trace ends where their grants do. A + pinned answer, by version here or by job on the run route, is one + project's. + + Its own endpoint rather than a field on the asset graph: the graph is + folder-wide and polled by a run page, while this is rendered for one + selection at a time. Empty for projects that did not opt into the + analysis pass (`column_lineage: true`), which is the ordinary case. The + indirect `scan` kind is stored but never served: it reaches every output + column of its model. + operationId: getDbtColumnLineage + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. + schema: + type: array + items: + type: string + - name: dbt_script_hash + in: query + description: > + The deployed version a view is drawing, when it is drawing one — the + dbt editor, which shows a single project as of a single deploy. A + version-pinned answer is that version's project alone, the same as a + job-pinned one, and only the unpinned answer crosses projects: a pin + says which stored graph is on screen, and another project's live + graph is not part of it. + + A run's or an editor buffer's own graph is not reachable here: that + pins to a job, and costs the job-read gate — see + `jobs/dbt_column_lineage/{id}`. + schema: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/assets/macros: get: summary: List every workspace DuckDB macro (deployed `// macros` libraries) @@ -24555,6 +24698,49 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/jobs/dbt_column_lineage/{id}: + get: + summary: Get relations' project column lineage as one run saw it + description: > + The same answer as `assets/column_lineage`, for the project version a + single job ran — including the dbt editor's parse of its own buffer, + whose graph belongs to that job and is reachable no other way. One + project answers here, the one the run is of, since the graph this + annotates is that project's too. Authorized through the job, the same + gate as `dbt_graph`. Reaching the run is not on its own enough to read + the project: a caller with no access to the script gets its relations and + `ref()` edges from `dbt_graph` and an empty answer here, exactly as that + endpoint redacts the model's SQL. + operationId: getDbtRunColumnLineage + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: The job whose graph the lineage is read from + schema: + type: string + format: uuid + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. + schema: + type: array + items: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/jobs/run_progress/{id}: get: summary: List the per-relation progress one job has recorded so far @@ -26169,6 +26355,40 @@ components: drawn identically and the ambiguity would otherwise just move into the editor. Omitted for the unpinned workspace graph, which spans every project and so has no one time. + DbtColumnLineage: + type: object + description: >- + The direct column-to-column lineage the asked-for relations' columns sit + in — the connected component around them — in the terms the canvas draws: + relations and columns, never dbt's node ids. + required: [edges, truncated] + properties: + edges: + type: array + items: + type: object + required: [from_asset_path, from_column, to_asset_path, to_column, kind] + properties: + from_asset_path: + type: string + from_column: + type: string + to_asset_path: + type: string + to_column: + type: string + kind: + type: string + description: >- + dbt's own word for how the value travelled — `copy` + (passthrough) or `mod` (transformed). Not an enum: the engine + treats the set as open. + truncated: + type: boolean + description: >- + The component reaches further than `edges`, which holds the part + nearest the asked-for relations. A trace that stops short is + otherwise indistinguishable from one that ends. DbtAssetProvenance: type: object description: >- @@ -26216,7 +26436,23 @@ components: columns: type: object additionalProperties: true - description: Declared column metadata (name -> description). NOT column lineage — `manifest.json` carries none. + description: Declared column metadata (name -> description) — what `manifest.json` carries, which is only the columns an author wrote down. Omitted when the caller cannot read the script. + column_schema: + type: array + description: >- + Every column of the relation, typed and in the order the model + produces them, from the engine's static analysis. Present only for a + project that opted into it, and gated like `columns` and the model's + SQL: a full column list is the shape of what the author wrote. + items: + type: object + required: [name] + properties: + name: + type: string + type: + type: string + description: The declared type where `schema.yml` gives one, else the inferred one. Omitted when neither is known. freshness: type: object additionalProperties: true diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index a674e6c38a..36b6eab0bb 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -5825,6 +5825,9 @@ mod embed_token_tests { "GET", ), ("/api/w/test/resources/list_search", "GET"), + // The metadata allowlist matches `resources/type/` by prefix, so a route + // added under it that is not a read must be denied by its method. + ("/api/w/test/resources/type/hub/pick/slack", "POST"), // Workspace-wide job enumeration/export must NOT be reachable — an app // reads only jobs it launched, by id (blocked via the app_embed sentinel). ("/api/w/test/jobs/list", "GET"), diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index dbf33cdddc..83a92bcd3c 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -139,6 +139,7 @@ pub fn workspaced_service() -> Router { .route("/run_progress/{id}", get(get_run_progress)) .route("/run_assets/{id}", get(list_run_assets)) .route("/dbt_graph/{id}", get(get_dbt_run_graph)) + .route("/dbt_column_lineage/{id}", get(get_dbt_run_column_lineage)) .route("/dbt_resumable/{id}", get(get_dbt_resumable)) .route( "/dbt_resumable_script/p/{*script_path}", @@ -891,21 +892,27 @@ struct AssetProgress { error: Option, } -/// The asset graph as one run saw it. Pinning to a job needs the full job-read -/// contract, so it lives on `require_job_read_access` here rather than as a -/// parameter on `/assets/graph`. See docs/dbt-runtime.md. -async fn get_dbt_run_graph( - authed: ApiAuthed, - OptViewToken(view_token): OptViewToken, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Query(q): Query, -) -> error::JsonResult { +/// Which project version a dbt view pins to for this job, once the caller has +/// been shown to be entitled to it. +/// +/// `Ok(None)` is "answer unpinned", not a refusal: a job that stored no graph of +/// its own — and one that has aged out of retention — is served the deployed +/// version rather than an error, so a run page keeps drawing after the run is +/// gone. Pinning needs the full job-read contract, which is why it lives on +/// `require_job_read_access` here rather than as a parameter on `/assets/*`. +/// See docs/dbt-runtime.md. +async fn dbt_pinned_run( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + job_id: Uuid, + view_token: Option<&str>, +) -> error::Result> { // The scope domain comes from the URL segment, so `/jobs` asks a scoped token // for `jobs:read` alone while the body returned is asset data. Both are // required: the job gate below reaches this run, this reaches assets at all. - check_scopes(&authed, || "assets:read".to_string())?; + check_scopes(authed, || "assets:read".to_string())?; let job = sqlx::query!( r#"SELECT created_by, runnable_path, CASE WHEN kind = 'script' THEN runnable_id END AS script_hash, @@ -918,42 +925,70 @@ async fn get_dbt_run_graph( AND g.script_hash IS NULL) AS "editor_graph!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, job_id, - &w_id + w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await?; - // No such job: answer the unpinned graph rather than 404, so a run page whose - // job has aged out of retention still draws the deployed version instead of - // an error. Reachable only with `assets:read`, which is exactly what - // `/assets/graph` would have cost for the same answer. + // Unpinned rather than 404 for a job that is gone. Reachable only with + // `assets:read`, which is exactly what the unpinned route would have cost + // for the same answer. let Some(job) = job else { - return windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, None).await; + return Ok(None); }; require_job_read_access( - &db, - &user_db, - &authed, - &w_id, + db, + user_db, + authed, + w_id, &job_id, &job.created_by, - view_token.as_deref(), + view_token, ) .await?; // A preview or flow job names no deployed version, so there is usually no // graph to pin to and the workspace one answers. The exception is a job that // parsed one itself, which is what the dbt editor's refresh is: its graph // belongs to that job alone and nothing else can reach it. - let pinned = job + Ok(job .runnable_path .filter(|_| job.script_hash.is_some() || job.editor_graph) .map(|path| windmill_api_assets::PinnedRun { job_id, script_path: path, script_hash: job.script_hash, - }); + })) +} + +/// The asset graph as one run saw it. +async fn get_dbt_run_graph( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(q): Query, +) -> error::JsonResult { + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, pinned).await } +/// The column lineage a set of relations sits in as one run saw it — the same +/// pin as `get_dbt_run_graph`, for the trace drawn beside a node of that graph. +async fn get_dbt_run_column_lineage( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(pairs): Query>, +) -> error::JsonResult { + let q = windmill_api_assets::ColumnLineageQuery::from_query_pairs(pairs)?; + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; + windmill_api_assets::dbt_column_lineage_for(&authed, &w_id, user_db, q, pinned).await +} + /// Whether a `dbt retry` submitted by this caller would resume THIS run. /// /// One failure is saved per script per execution principal, so a page showing an diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 05cd354cd4..1c22981ee0 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -346,7 +346,7 @@ pub(crate) struct ArchiveQueryParams { default_ts: Option, /// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format settings_version: Option, - /// Opt-in: include `extra_perms` on flow / script / app rows. Default `false` + /// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false` /// so cross-workspace tarball imports do not carry over ACLs referring to /// identities that may not exist in the target workspace. `wmill sync pull` /// passes `true` to surface ACLs in the git-tracked yaml. @@ -365,8 +365,8 @@ pub(crate) struct ArchiveQueryParams { /// pre-existing serialization for folders and groups so /// no customer sees a one-time noisy diff on upgrade. /// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}` -/// or null. New surface for flow / script / app, which -/// never carried ACLs in source before this change. +/// or null. New surface for script / flow / app / variable, +/// which never carried ACLs in source before this change. #[derive(Clone, Copy)] pub enum ExtraPermsBehavior { Drop, @@ -665,7 +665,7 @@ pub(crate) async fn tarball_workspace( check_scopes(&authed, || "variables:read".to_string())?; } - // Opt-in behavior for surfacing per-resource ACLs on flow/app rows. + // Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows. // Folder and group rows have always carried `extra_perms` in source and // continue to do so unconditionally (`KeepEvenEmpty`) so existing // customer git repos see no one-time noisy diff. @@ -1002,8 +1002,7 @@ pub(crate) async fn tarball_workspace( Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) })?); } - let var_str = - &to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap(); + let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) .await?; diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 117b41632a..4d5523524c 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -175,10 +175,12 @@ fn is_write_access(access: Option) -> bool { /// producers). Resource / datatable / volume reads stay explicit-`// on`: /// a config/lookup read cascading is more often surprising than wanted. fn is_auto_trigger_kind(kind: AssetKind) -> bool { - // `Dbt` is deliberately NOT here. dbt is the only thing that can produce a - // warehouse relation (`// materialize` takes DuckLake targets only) and a dbt - // run does not dispatch, so a derived `dbt://` edge could never fire — it - // would draw a cascade arrow into a script nothing can wake. + // `Dbt` is deliberately NOT here. A warehouse relation is usually built by + // the dbt project that reads it, and a dbt run does not dispatch, so deriving + // an edge from every `dbt://` read would draw cascade arrows that mostly never + // fire. The relations a native `// materialize manual dbt://…` script writes + // do wake subscribers, but only through an explicit `// on`, which is where + // the author states that this particular relation has such a producer. matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) } @@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs( out } +/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is +/// its ONLY producer. +/// +/// That is the one shape in which subscribing to a warehouse relation can never +/// be woken: a dbt run records the models it built and does not dispatch +/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that +/// declares `// materialize manual dbt://…` fans out on the ordinary path. +/// +/// `None` covers both "some non-dbt script materializes it" and "nothing +/// produces it yet" — the second is the ordinary deploy-order case, identical to +/// every other asset kind, not a dormant edge. +/// +/// **Give it the workspace pool, not an RLS-scoped transaction.** `script` +/// carries RLS while `asset` does not, so a scoped executor hides producers, and +/// the hidden ones fail in the harmful direction: a native producer the deployer +/// cannot read leaves a dbt-only set behind and refuses a subscription that would +/// have fired. What it discloses in exchange is the path of a dbt script building +/// a relation the caller already named, which the workspace asset graph hands out +/// for every `dbt://` node anyway (the source that script wrote stays gated). +/// Callers must therefore already be scoped to `workspace_id`. +/// +/// `deploying_paths` is excluded from the producer set, and has to be: reading +/// committed rows means the deploying script's own are the version being +/// replaced, so one that just dropped its `// materialize` would still count as a +/// producer and let a now-dormant subscription through. Pass every path this +/// deploy is rewriting — under a rename that is the old path as well as the new +/// one, whose committed write row the transaction is about to remove. Excluding +/// them is free of the opposite error, because a script never wakes its own +/// subscription — the dispatcher skips that as a self-loop. +/// +/// A producer another deploy is committing concurrently is invisible either way, +/// and the outcome depends on which side it is. An uncommitted NATIVE producer +/// leaves a dbt-only set and refuses, with a message the user can retry past. An +/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then +/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger +/// row lands, neither side reports the edge it left dormant. Serializing the two +/// is not worth it: they would have to share a per-relation lock, and the ingest +/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding +/// relation locks first inverts that order into a deadlock across the two +/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md). +pub async fn sole_dbt_producer<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_path: &str, + deploying_paths: &[String], +) -> error::Result> { + use crate::scripts::ScriptLang; + let producers = sqlx::query!( + r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang" + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2 + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND a.usage_path <> ALL($3)"#, + workspace_id, + asset_path, + deploying_paths + ) + .fetch_all(executor) + .await?; + if producers + .iter() + .any(|p| !matches!(p.language, ScriptLang::Dbt)) + { + return Ok(None); + } + Ok(producers.into_iter().next().map(|p| p.path)) +} + +/// The set form of [`sole_dbt_producer`], for asking about many relations at +/// once: every `// on dbt://` edge among `relations` whose producers +/// are all dbt scripts, rendered as `dbt:// → `. +/// +/// A dbt deploy asks this about the relations it just ingested, because that +/// ingest is what can retroactively leave a subscription accepted earlier — when +/// nothing produced the relation — with dbt as its only producer. +/// +/// Spells the predicate the same way its singular sibling does, per subscriber: +/// the producer set excludes the subscriber's own path (a script never wakes +/// itself) and has to be non-empty (nothing produces it yet is deploy order, not +/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would +/// silence this warning with nothing failing. +/// +/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace +/// pool, caller already scoped to `workspace_id`. +pub async fn dormant_dbt_subscriptions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + relations: &[String], +) -> error::Result> { + if relations.is_empty() { + return Ok(vec![]); + } + let refs = relations + .iter() + .map(|r| format!("dbt://{r}")) + .collect::>(); + Ok(sqlx::query_scalar!( + r#"WITH producer AS ( + SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND 'dbt://' || a.path = ANY($2) + ) + SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!" + FROM script_trigger st + WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset' + AND st.trigger_ref = ANY($2) + AND EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language = 'dbt') + AND NOT EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language <> 'dbt') + ORDER BY 1"#, + workspace_id, + &refs + ) + .fetch_all(executor) + .await?) +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index 8fb124a33f..157a61f129 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -8,9 +8,11 @@ //! Two things this module is deliberate about: //! //! * **Asset identity is the physical relation.** A model becomes -//! `dbt:////`: the scheme names dbt, which is the -//! only thing that creates one, but the PATH is the relation and never dbt's -//! own `unique_id`. Two projects meet at a handoff — one materializes a mart, +//! `dbt:////`: the scheme names the namespace dbt +//! made — it is the only thing that DERIVES one, while any other language can +//! declare a write to one (decision 25) — but the PATH is the relation and +//! never dbt's own `unique_id`. Two projects meet at a handoff — one +//! materializes a mart, //! the next declares it a `source` — where `model.a.orders` and //! `source.b.analytics.orders` differ but the relation does not, so keying on //! the node id would leave each project an island; a native script reading the @@ -33,8 +35,8 @@ //! Every `pub` mutator in this module — the manifest ones //! (`replace_dbt_manifest`, `clear_dbt_manifest_version`, //! `clear_dbt_editor_graphs`), -//! the snapshot sweep, and the retry-state ones (`move_dbt_run_state`, -//! `clear_dbt_run_state`, `clear_dbt_run_state_if_path_retired`) — takes the +//! the snapshot sweep, and the script-state ones (`move_dbt_script_state`, +//! `clear_dbt_script_state`, `clear_dbt_script_state_if_path_retired`) — takes the //! workspace and the script to act on as plain arguments and enforces nothing: //! **the caller must already have verified write access to that script**, //! exactly like the sibling `assets::replace_static_asset_usage` each is called @@ -188,6 +190,19 @@ fn graph_digest(ingested: &IngestedManifest, relation_root: &str) -> String { .unwrap_or_default() .as_bytes(), ); + // Only when there are any, so a project that never asked for the analysis + // pass keeps the digest it already has. Hashing an empty section + // unconditionally would change every stored digest at once, and every + // dynamic run would then store a full snapshot until its script is + // redeployed — which reads exactly like the suppression above never working. + if !ingested.column_edges.is_empty() { + h.update(b"\0"); + h.update( + serde_json::to_string(&ingested.column_edges) + .unwrap_or_default() + .as_bytes(), + ); + } format!("{:x}", h.finalize()) } @@ -297,6 +312,14 @@ pub async fn prune_dbt_run_graphs( ) .execute(db) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE job_id <> '00000000-0000-0000-0000-000000000000' + AND ingested_at < now() - make_interval(days => $1)", + RUN_GRAPH_RETENTION_DAYS, + ) + .execute(db) + .await?; // In ONE transaction with the orphan sweep: a restart in the gap leaves graph // rows whose marker is gone, and since the sweep runs only when a marker went, // every later call computes `retired == 0` and skips them for good. @@ -325,7 +348,7 @@ pub async fn prune_dbt_run_graphs( // partial index here — all of them `WHERE job_id <> DEPLOYED` — and past the // keep-count is rare, so the ordinary run should pay for neither. if retired > 0 { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} t WHERE t.workspace_id = $1 AND t.script_path = $2 @@ -381,6 +404,18 @@ pub struct IngestedNode { pub severity: Option, pub attached_node: Option, pub columns: Option, + /// The node's real columns, typed and ordered — `[{"name": …, "type": …}]`, + /// from the engine's static analysis. `None` when the project did not ask + /// for it or the engine wrote none. Beside `columns` rather than merged into + /// it: that one is what the author DECLARED, and stays that. + /// + /// Skipped when absent, unlike its neighbours, because `graph_digest` + /// serializes these nodes: emitting `"column_schema":null` would change + /// every stored digest at once, and every dynamic run of a project that + /// never asked for the pass would store a full snapshot until its script is + /// redeployed. + #[serde(skip_serializing_if = "Option::is_none")] + pub column_schema: Option, pub freshness: Option, /// The transform itself, for the graph to render. The copy taken at /// deploy: the file itself is in the script's module bundle. @@ -388,6 +423,40 @@ pub struct IngestedNode { pub original_file_path: Option, } +/// One column-to-column edge of the ingested graph. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +#[serde(default)] +pub struct IngestedColumnEdge { + pub parent_unique_id: String, + pub parent_column: String, + pub child_unique_id: String, + pub child_column: String, + /// dbt's own word: `copy`, `mod` or `scan`. Kept verbatim — the engine's own + /// reader maps those three and passes anything else through, so the set is + /// open. + pub lineage_kind: String, +} + +/// One column of a node, as the engine's static analysis resolved it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexedColumn { + pub name: String, + /// The declared type where `schema.yml` gives one, else the inferred one. + /// Empty when neither is known; the column still belongs to the relation, so + /// only the type is left out. + pub column_type: String, + /// Position in the relation, which is the order the panel lists them in. + pub index: i64, +} + +/// What one `--write-index` pass produced: the column edges of the whole +/// project and the real column schema per node. +#[derive(Debug, Default)] +pub struct ColumnIndex { + pub edges: Vec, + pub columns: HashMap>, +} + // Serde: an agent worker cannot write these tables directly, so it posts the // whole manifest to the server, which stores it with the same function the SQL // path uses. @@ -398,6 +467,9 @@ pub struct IngestedNode { pub struct IngestedManifest { pub nodes: Vec, pub edges: Vec<(String, String)>, + /// Column-to-column lineage, when the project asked for it and the engine + /// produced it. Empty is the normal case — see `attach_column_index`. + pub column_edges: Vec, /// The `asset` rows the owning script produces (models) and consumes /// (sources) — what the lineage graph is drawn from. pub assets: Vec, @@ -405,6 +477,88 @@ pub struct IngestedManifest { pub adapter_type: String, } +/// The most column edges one graph stores. +/// +/// A `scan` edge — the column was read to produce the row, not the value — is +/// emitted from every join key and every predicate column to every output +/// column, so one wide model over a multi-column join contributes columns times +/// predicates edges on its own. The cap is what keeps a project shaped like that +/// from turning one deploy into a multi-million-row insert; past it the lineage +/// is truncated and the rest of the graph is unaffected. +pub const MAX_COLUMN_EDGES: usize = 200_000; + +/// Whether the value travelled along this edge, as opposed to the column merely +/// being read to produce the row. +/// +/// A `scan` edge reaches every output column of its model, so it is most of what +/// a wide project's index holds and the first thing `MAX_COLUMN_EDGES` gives up. +/// It is still stored, for a view that wants indirect influence. +pub fn is_direct(lineage_kind: &str) -> bool { + matches!(lineage_kind, "copy" | "mod") +} + +impl IngestedManifest { + /// Fold one `--write-index` pass into the graph. + /// + /// Both halves are scoped to the nodes this graph already kept: the index + /// describes the whole project, while the graph describes what this script's + /// selection builds plus the parents anchoring its edges, and an edge whose + /// endpoint is absent has nothing to draw. + pub fn attach_column_index(&mut self, index: ColumnIndex) { + let kept: std::collections::HashSet<&str> = + self.nodes.iter().map(|n| n.unique_id.as_str()).collect(); + let mut edges: Vec = index + .edges + .into_iter() + .filter(|e| { + kept.contains(e.parent_unique_id.as_str()) + && kept.contains(e.child_unique_id.as_str()) + }) + .collect(); + // Sorted and deduplicated for the digest, which decides whether a run + // stores a snapshot at all: parquet row order is the engine's and two + // passes over one project must not read as two different graphs. + // + // Direct kinds first, so what the truncation below gives up is `scan` — + // the bulk of a wide project's lineage, and the kind that says the column + // was read to produce the row rather than the value. The + // worker's reader already applies this order while decoding, because the + // memory bound has to; repeating it here is what makes the ordering a + // property of the manifest rather than of one caller's reader, and it is + // the only ordering an index assembled some other way would get. + edges.sort_by(|a, b| { + is_direct(&b.lineage_kind) + .cmp(&is_direct(&a.lineage_kind)) + .then_with(|| a.cmp(b)) + }); + edges.dedup(); + edges.truncate(MAX_COLUMN_EDGES); + self.column_edges = edges; + + let mut columns = index.columns; + for node in self.nodes.iter_mut() { + let Some(mut cols) = columns.remove(&node.unique_id) else { + continue; + }; + if cols.is_empty() { + continue; + } + cols.sort_by_key(|c| c.index); + node.column_schema = Some(serde_json::Value::Array( + cols.into_iter() + // A column the analysis typed as nothing still belongs in + // the list — that it exists is the half `manifest.json` + // could not answer. + .map(|c| match c.column_type.is_empty() { + true => serde_json::json!({ "name": c.name }), + false => serde_json::json!({ "name": c.name, "type": c.column_type }), + }) + .collect(), + )); + } + } +} + /// dbt's `materialized` mapped onto Windmill's write strategy. /// /// The mapping is exact for the four strategies Windmill has, and deliberately @@ -655,6 +809,9 @@ pub fn ingest_manifest( .map(|(k, v)| (k.clone(), v.description.clone().unwrap_or_default())) .collect::>()) }), + // Filled by `attach_column_index` when the project asked for it: + // the manifest carries declared columns only. + column_schema: None, freshness: node.freshness.clone(), // The transform the graph renders. Capped: a project can hold // thousands of models and this is duplicated per deploy, so a @@ -826,6 +983,16 @@ pub async fn replace_dbt_manifest( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2 + AND script_hash = $3 AND job_id = $4", + workspace_id, + script_path, + script_hash, + job_id + ) + .execute(&mut **tx) + .await?; // The marker, before the rows: a graph with no nodes at all is a legitimate // answer for a dynamic run that disabled every model, and the reader must be // able to tell it from a run that stored nothing. @@ -881,7 +1048,7 @@ async fn insert_graph_rows( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, \ resource_type, name, asset_path, materialized, materialize_strategy, unique_key, \ tags, description, test_kind, test_column, test_args, severity, attached_node, \ - columns, freshness, raw_code, original_file_path) ", + columns, column_schema, freshness, raw_code, original_file_path) ", ); q.push_values(chunk, |mut b, n| { b.push_bind(workspace_id) @@ -903,6 +1070,7 @@ async fn insert_graph_rows( .push_bind(&n.severity) .push_bind(&n.attached_node) .push_bind(&n.columns) + .push_bind(&n.column_schema) .push_bind(&n.freshness) .push_bind(&n.raw_code) .push_bind(&n.original_file_path); @@ -926,6 +1094,26 @@ async fn insert_graph_rows( q.push(" ON CONFLICT DO NOTHING"); q.build().execute(&mut **tx).await?; } + + for chunk in ingested.column_edges.chunks(COLUMN_EDGE_INSERT_CHUNK) { + let mut q = sqlx::QueryBuilder::new( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, \ + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind) ", + ); + q.push_values(chunk, |mut b, e| { + b.push_bind(workspace_id) + .push_bind(script_path) + .push_bind(script_hash) + .push_bind(job_id) + .push_bind(&e.parent_unique_id) + .push_bind(&e.parent_column) + .push_bind(&e.child_unique_id) + .push_bind(&e.child_column) + .push_bind(&e.lineage_kind); + }); + q.push(" ON CONFLICT DO NOTHING"); + q.build().execute(&mut **tx).await?; + } Ok(()) } @@ -966,7 +1154,7 @@ pub async fn replace_dbt_editor_graph( ) -> Result<()> { // By job alone, so re-executing one — a zombie recovered onto another // worker — replaces its rows rather than colliding with them. - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = $2 AND script_hash IS NULL" )) @@ -1016,7 +1204,7 @@ pub async fn replace_dbt_editor_graph( .fetch_all(&mut **tx) .await?; if !retired.is_empty() { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = ANY($2) \ AND script_hash IS NULL" @@ -1035,6 +1223,8 @@ pub async fn replace_dbt_editor_graph( const NODE_INSERT_CHUNK: usize = 2000; /// Six columns, so the same ceiling allows far more. const EDGE_INSERT_CHUNK: usize = 8000; +/// Nine columns, and by far the most numerous rows of the three. +const COLUMN_EDGE_INSERT_CHUNK: usize = 6000; /// Clear one VERSION's graph: the delete-by-hash route, which only soft-deletes /// its `script` row and so fires no cascade, and the ingest that finds no @@ -1070,6 +1260,15 @@ pub async fn clear_dbt_manifest_version( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + workspace_id, + script_path, + script_hash + ) + .execute(&mut **tx) + .await?; // The marker too, and every job's: a marker left standing for rows that are // gone is read as a snapshot, and its digest still answers the suppression // check — so an identical run would write nothing and then render an empty @@ -1105,7 +1304,7 @@ pub async fn clear_dbt_editor_graphs( workspace_id: &str, script_path: &str, ) -> Result<()> { - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL" @@ -1118,22 +1317,27 @@ pub async fn clear_dbt_editor_graphs( Ok(()) } -/// Move a dbt script's saved retry state to its new path. +/// Move a dbt script's saved state to its new path: the run `dbt retry` resumes, +/// and the state each environment's deferrals resolve through. /// -/// Keyed by path like the sidecar, but unlike the sidecar it is not -/// regenerated by anything: the deploy re-ingests a manifest, while these are -/// the results of a run that already happened. Clearing on rename would throw -/// away a resumable failure for a cosmetic change, so it travels instead. +/// Keyed by path like the sidecar, but unlike the sidecar neither is regenerated +/// by anything: the deploy re-ingests a manifest, while these are the results of +/// runs that already happened. Clearing on rename would throw away a resumable +/// failure, and every deferral until the next full run, for a cosmetic change — +/// so they travel instead. An artifact too large for its row is unaffected: its +/// key is that publication's own, and the moved row is what names it. /// /// See the mutator contract above: this authorizes nothing. -pub async fn move_dbt_run_state( +pub async fn move_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, old_path: &str, new_path: &str, ) -> Result<()> { // The destination may already hold state from a script that lived there - // before; the incoming row is the newer truth for this project. + // before; the incoming row is the newer truth for this project. What the + // displaced row named in object storage is left there, as a cleared one's is + // — see `clear_dbt_script_state`. sqlx::query!( "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2", workspace_id, @@ -1149,22 +1353,38 @@ pub async fn move_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + new_path + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE dbt_environment_state SET script_path = $3 + WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + old_path, + new_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop the saved retry state, but only once NO live version of the path is -/// left. +/// Drop the saved state, but only once NO live version of the path is left. /// -/// `dbt_run_state`'s key is the path and the principal — one saved run per script -/// per identity it executes as, not -/// one per version — so archiving or deleting a single version must not take it -/// with them: the live version's `dbt retry` would be refused and the -/// partial-failure resume lost. It does not need to be version-scoped either, -/// because `identity` already refuses a resume whose project, warehouse or -/// engine moved. +/// Neither table is keyed by version — `dbt_run_state` by path and principal, +/// `dbt_environment_state` by path and environment — so archiving or deleting a +/// single version must not take them with it: the live version's `dbt retry` +/// would be refused, its partial-failure resume lost, and every deferral would +/// have to wait for another full run to republish. Neither needs to be +/// version-scoped either: `identity` already refuses a resume whose project, +/// warehouse or engine moved, and a deferral resolves relation names, which a +/// new version of the same project spells the same way. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state_if_path_retired( +pub async fn clear_dbt_script_state_if_path_retired( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1179,17 +1399,34 @@ pub async fn clear_dbt_run_state_if_path_retired( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2 + AND NOT EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false)", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop a dbt script's saved retry state. +/// Drop a dbt script's saved state, both halves. /// -/// Archive and delete: `run_results` is not small, the invocation arguments it -/// carries are the user's, and a script later created at the same path would -/// otherwise inherit a stranger's resumable failure. +/// Archive and delete: neither is small, the invocation arguments and manifest +/// they carry are the user's, and a script later created at the same path would +/// otherwise inherit a stranger's resumable failure and defer to a project it +/// has nothing to do with. +/// +/// An artifact too large for its row lives in the instance's object storage, and +/// this leaves it there — as a deleted script leaves its bundle. Reaching it from +/// here would mean an object-store client in this crate and a delete that has to +/// land after the caller's transaction commits, for one object per environment of +/// a script that is gone. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state( +pub async fn clear_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1201,6 +1438,13 @@ pub async fn clear_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } @@ -1765,6 +2009,69 @@ mod tests { assert_eq!(back.assets.len(), ingested.assets.len()); assert_eq!(back.assets[0].path, ingested.assets[0].path); } + + // The index describes the whole PROJECT while the graph describes what this + // script's selection builds, so an edge whose endpoint the graph does not + // hold has nothing to draw and must not be stored. + #[test] + fn column_lineage_is_scoped_to_the_nodes_the_graph_kept() { + let mut i = ingested(); + let kept = "model.jaffle_shop.customers"; + let dropped = "model.other_project.elsewhere"; + i.attach_column_index(ColumnIndex { + edges: vec![ + edge("model.jaffle_shop.orders_daily", "id", kept, "id", "copy"), + edge(dropped, "id", kept, "id", "copy"), + edge(kept, "id", dropped, "id", "copy"), + ], + columns: [ + ( + kept.to_string(), + vec![ + col("total", "Float64", 1), + col("id", "Int32", 0), + col("untyped", "", 2), + ], + ), + (dropped.to_string(), vec![col("id", "Int32", 0)]), + ] + .into(), + }); + assert_eq!( + i.column_edges + .iter() + .map(|e| (e.parent_unique_id.as_str(), e.child_unique_id.as_str())) + .collect::>(), + vec![("model.jaffle_shop.orders_daily", kept)] + ); + // In `column_index` order, and a column the analysis could not type still + // belongs to the relation. + assert_eq!( + node(&i, kept).column_schema, + Some(serde_json::json!([ + {"name": "id", "type": "Int32"}, + {"name": "total", "type": "Float64"}, + {"name": "untyped"}, + ])) + ); + assert!(node(&i, "model.jaffle_shop.orders_daily") + .column_schema + .is_none()); + } + + fn edge(from: &str, from_col: &str, to: &str, to_col: &str, kind: &str) -> IngestedColumnEdge { + IngestedColumnEdge { + parent_unique_id: from.into(), + parent_column: from_col.into(), + child_unique_id: to.into(), + child_column: to_col.into(), + lineage_kind: kind.into(), + } + } + + fn col(name: &str, column_type: &str, index: i64) -> IndexedColumn { + IndexedColumn { name: name.into(), column_type: column_type.into(), index } + } } /// Record one model's state for THIS RUN. diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 115dae34e7..aed9b5c5de 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -401,6 +401,45 @@ pub async fn fetch_draft_only_list_rows( Ok(rows) } +/// Delete the caller's OWN draft at a path with no deployed row, for the DELETE +/// route of a kind whose list synthesizes such rows via +/// `fetch_draft_only_list_rows`. The `NOT EXISTS` leaves a deployed row's draft +/// alone, so a route may call this on its not-found branch whatever the reason +/// for the miss. `Ok(false)` means nothing matched: the caller reports its own error. +/// +/// Takes no permission check and callers must not add one: an email-scoped row +/// belongs to the caller, who can always discard it, as `update_draft`'s +/// own-discard does. Legacy (`email IS NULL`) rows are owned by nobody and keep +/// their write gate, so discarding one stays on the `update_draft` route. +pub async fn delete_draft_only_for_path( + db: &DB, + w_id: &str, + kind: UserDraftItemKind, + path: &str, + email: &str, +) -> Result { + let Some(table) = kind.deployed_table() else { + return Ok(false); + }; + // `table` is from the closed `deployed_table()` enum, never user input. + let sql = format!( + "DELETE FROM draft \ + WHERE workspace_id = $1 AND typ = $2::text::DRAFT_KIND AND path = $3 \ + AND email = $4 \ + AND NOT EXISTS (SELECT 1 FROM {table} t \ + WHERE t.workspace_id = draft.workspace_id AND t.path = draft.path)" + ); + let deleted = sqlx::query(&sql) + .bind(w_id) + .bind(kind.as_str()) + .bind(path) + .bind(email) + .execute(db) + .await? + .rows_affected(); + Ok(deleted > 0) +} + /// The get-by-path draft choreography, shared by every entity's "get by path" /// route. Given the deployed entity as an `Option` (caller maps its own "not /// found" to `None`): diff --git a/backend/windmill-common/tests/dbt_graph_storage.rs b/backend/windmill-common/tests/dbt_graph_storage.rs index 7b4971312a..3226d27e9e 100644 --- a/backend/windmill-common/tests/dbt_graph_storage.rs +++ b/backend/windmill-common/tests/dbt_graph_storage.rs @@ -7,9 +7,10 @@ use sqlx::{Pool, Postgres}; use windmill_common::dbt_manifest::{ - clear_dbt_editor_graphs, clear_dbt_manifest_version, prune_dbt_run_graphs, - replace_dbt_editor_graph, replace_dbt_manifest, IngestedManifest, IngestedNode, - DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, + clear_dbt_editor_graphs, clear_dbt_manifest_version, clear_dbt_script_state, + clear_dbt_script_state_if_path_retired, move_dbt_script_state, prune_dbt_run_graphs, + replace_dbt_editor_graph, replace_dbt_manifest, IngestedColumnEdge, IngestedManifest, + IngestedNode, DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, }; const WS: &str = "test-workspace"; @@ -52,10 +53,36 @@ fn manifest(names: &[&str]) -> IngestedManifest { .windows(2) .map(|w| (format!("model.p.{}", w[0]), format!("model.p.{}", w[1]))) .collect(), + // Same reason: a project that opted into the analysis pass has these, and + // a fixture without them leaves every column-edge insert and sweep in + // this file unexecuted. + column_edges: names + .windows(2) + .map(|w| IngestedColumnEdge { + parent_unique_id: format!("model.p.{}", w[0]), + parent_column: w[0].to_string(), + child_unique_id: format!("model.p.{}", w[1]), + child_column: w[1].to_string(), + lineage_kind: "copy".to_string(), + }) + .collect(), ..Default::default() } } +/// Column edges of one version, so the sweeps can be shown to reach them. +async fn column_edges_for(db: &Pool, hash: i64) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + WS, + hash + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} + /// Edges for one version, so a test can assert the batched insert ran at all. async fn edges_for(db: &Pool, hash: i64) -> i64 { sqlx::query_scalar!( @@ -135,6 +162,33 @@ async fn an_identical_run_stores_no_snapshot(db: Pool) { assert_eq!(markers(&db, 1).await, 1, "and leaves no marker of its own"); } +/// A column that is projected AND used as a predicate for the same output column +/// has both a `copy` edge and a `scan` one. They are two facts, and the digest +/// counts both — so the uniqueness key has to carry `lineage_kind`, or the +/// second is dropped by `ON CONFLICT DO NOTHING` while the digest still claims +/// it was stored. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn both_kinds_of_one_column_pair_are_stored(db: Pool) { + deploy_script(&db, 1).await; + let pair = |kind: &str| IngestedColumnEdge { + parent_unique_id: "model.p.a".to_string(), + parent_column: "id".to_string(), + child_unique_id: "model.p.b".to_string(), + child_column: "id".to_string(), + lineage_kind: kind.to_string(), + }; + let mut m = manifest(&["a", "b"]); + m.column_edges = vec![pair("copy"), pair("scan")]; + + let mut tx = db.begin().await.unwrap(); + replace_dbt_manifest(&mut tx, WS, PATH, 1, None, &m, "root") + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!(column_edges_for(&db, 1).await, 2, "both kinds survive"); +} + /// A run whose model set differs keeps its own, and the version's is untouched: /// this is what lets an older run page render the project that run built. #[sqlx::test(migrations = "../migrations", fixtures("base"))] @@ -235,7 +289,11 @@ async fn clearing_one_version_leaves_the_others(db: Pool) { // this is where two versions coexist: it pins the batched edge insert // against a real database as well as the version scoping. assert_eq!(edges_for(&db, 1).await, 0, "the cleared version's edges go"); - assert_eq!(edges_for(&db, 2).await, 1, "the other version keeps its own"); + assert_eq!( + edges_for(&db, 2).await, + 1, + "the other version keeps its own" + ); } /// The routes that hard-delete a path clear no graph rows: they delete the @@ -275,6 +333,8 @@ async fn deleting_the_script_cascades_to_every_sidecar(db: Pool) { assert_eq!(nodes_for(&db, 2, DEPLOYED_GRAPH).await, 0); assert_eq!(edges_for(&db, 1).await, 0); assert_eq!(edges_for(&db, 2).await, 0); + assert_eq!(column_edges_for(&db, 1).await, 0); + assert_eq!(column_edges_for(&db, 2).await, 0); assert_eq!(markers_for_path(&db).await, 0); } @@ -315,7 +375,7 @@ async fn the_sweep_takes_old_snapshots_and_spares_the_version(db: Pool tx.commit().await.unwrap(); // Age one snapshot past the window, rows and marker together. - for t in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for t in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "UPDATE {t} SET ingested_at = now() - interval '400 days' WHERE job_id = $1" )) @@ -363,7 +423,11 @@ async fn only_the_newest_deploys_keep_their_graph(db: Pool) { // The newest is always among them: losing the live version's graph would // empty the page of every run of it. assert_eq!(nodes_for(&db, over, DEPLOYED_GRAPH).await, 1); - assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0, "the oldest is reclaimed"); + assert_eq!( + nodes_for(&db, 1, DEPLOYED_GRAPH).await, + 0, + "the oldest is reclaimed" + ); } /// The third provenance: a `parse` of the EDITOR's buffer, which names no @@ -480,17 +544,27 @@ async fn a_version_clear_spares_editor_graphs_and_a_path_clear_does_not(db: Pool replace_dbt_editor_graph(&mut tx, WS, PATH, job, ME, &manifest(&["a"]), "root") .await .unwrap(); - clear_dbt_manifest_version(&mut tx, WS, PATH, 1).await.unwrap(); + clear_dbt_manifest_version(&mut tx, WS, PATH, 1) + .await + .unwrap(); tx.commit().await.unwrap(); assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0); - assert_eq!(editor_nodes(&db, job).await, 1, "the buffer's graph survives"); + assert_eq!( + editor_nodes(&db, job).await, + 1, + "the buffer's graph survives" + ); let mut tx = db.begin().await.unwrap(); clear_dbt_editor_graphs(&mut tx, WS, PATH).await.unwrap(); tx.commit().await.unwrap(); - assert_eq!(editor_nodes(&db, job).await, 0, "retiring the path takes it"); + assert_eq!( + editor_nodes(&db, job).await, + 0, + "retiring the path takes it" + ); } /// A preview names its own PATH and needs only `jobs:run`, so a bound over the @@ -552,3 +626,157 @@ async fn editor_markers(db: &Pool) -> i64 { .unwrap() .unwrap_or(0) } + +/// A deferral resolves a `ref()` through the manifest of the last successful run +/// at this path, so that state has to follow the script the way the retry state +/// does: a rename must not strand it, and a path no live dbt version occupies +/// must not hand its manifest to whatever is created there next. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn environment_state_follows_the_script(db: Pool) { + const MOVED: &str = "f/test/renamed"; + deploy_script(&db, 1).await; + publish_environment_state(&db, PATH).await; + + let mut tx = db.begin().await.unwrap(); + move_dbt_script_state(&mut tx, WS, PATH, MOVED) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, PATH).await, 0); + assert_eq!(environment_states(&db, MOVED).await, 1); + + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state(&mut tx, WS, MOVED).await.unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, MOVED).await, 0); +} + +/// Archiving or deleting ONE version must not take the path's state with it — +/// the live version's next deferral still needs it — while the last one leaving +/// must, or a script later created at that path inherits the previous project's +/// manifest. The condition is a `NOT EXISTS` in raw SQL, so both directions are +/// pinned against a real database. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn state_goes_only_once_no_live_version_is_left(db: Pool) { + deploy_script(&db, 1).await; + deploy_script(&db, 2).await; + publish_environment_state(&db, PATH).await; + + retire(&db, 1).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 1, + "another version is still live here" + ); + + retire(&db, 2).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 0, + "the last one leaving takes it" + ); +} + +async fn retire(db: &Pool, hash: i64) { + sqlx::query!( + "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + WS, + hash + ) + .execute(db) + .await + .unwrap(); +} + +/// The worker publishes under a guard naming the version that ran, and the whole +/// point of it is a job that finishes late: its script can be renamed away and an +/// unrelated one created at the same path while it runs, and that project must +/// not inherit this one's manifest as its deferral state. Enforced in raw SQL, +/// where a refactor can drop a predicate with no type error, so it is pinned +/// against a real database — the same shape `dbt_state::publish` issues. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_late_job_cannot_publish_for_a_path_it_no_longer_owns(db: Pool) { + deploy_script(&db, 1).await; + assert_eq!(guarded_publish(&db, PATH, 1).await, 1, "its own version"); + assert_eq!( + guarded_publish(&db, PATH, 2).await, + 0, + "a version that never lived here" + ); + + // The script is gone from this path and another one takes it. + sqlx::query!( + "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + WS + ) + .execute(&db) + .await + .unwrap(); + deploy_script(&db, 3).await; + assert_eq!( + guarded_publish(&db, PATH, 1).await, + 0, + "the late job's version does not own this path any more" + ); +} + +/// The predicate `dbt_state::publish` locks the script row on, reduced to what it +/// decides. Keep the two in step — this file cannot call `publish` itself, which +/// is `pub(crate)` in `windmill-worker`. +async fn guarded_publish(db: &Pool, path: &str, ran: i64) -> u64 { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text + WHERE EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false + AND language = 'dbt' + AND (hash = $4 OR $4 = ANY(parent_hashes))) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + WS, + path, + uuid::Uuid::from_u128(9), + ran, + ) + .execute(db) + .await + .unwrap() + .rows_affected() +} + +async fn publish_environment_state(db: &Pool, path: &str) { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + WS, + path, + uuid::Uuid::from_u128(9), + ) + .execute(db) + .await + .unwrap(); +} + +async fn environment_states(db: &Pool, path: &str) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} diff --git a/backend/windmill-common/tests/dbt_producer_rules.rs b/backend/windmill-common/tests/dbt_producer_rules.rs new file mode 100644 index 0000000000..f63e0879b5 --- /dev/null +++ b/backend/windmill-common/tests/dbt_producer_rules.rs @@ -0,0 +1,196 @@ +/*! + * One rule, two spellings: "is dbt the sole producer of this warehouse + * relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription + * is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy + * retroactively leaves unwakeable. Both must answer "yes, dormant" only when + * every script writing the relation is a dbt one — a dbt run does not dispatch — + * and "no" both when a native `// materialize manual dbt://…` producer exists and + * when nothing produces the relation yet, which is the ordinary deploy-order + * case. Every way of getting this wrong is silent: a dormant edge on the canvas, + * a refused deploy of a valid pipeline, or a warning that stops appearing. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer}; + +const WS: &str = "test-workspace"; +const RELATION: &str = "main/analytics/orders"; +const SUBSCRIBER: &str = "u/test-user/consumer"; + +async fn plant_producer(db: &Pool, path: &str, language: &str, hash: i64) { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(language) + .execute(db) + .await + .expect("insert script"); + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING", + ) + .bind(WS) + .bind(RELATION) + .bind(path) + .execute(db) + .await + .expect("insert asset"); +} + +async fn plant_subscriber(db: &Pool, path: &str) { + sqlx::query( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, + trigger_ref) + VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)", + ) + .bind(WS) + .bind(path) + .bind(RELATION) + .execute(db) + .await + .expect("insert script_trigger"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn no_producer_is_not_dormant(db: Pool) { + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None, + "a relation nothing produces yet must not refuse the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn dbt_only_producer_is_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None + ); +} + +/// `asset` is keyed by path while `script` holds every version of it, so the +/// language has to be read off the live one: a path converted to dbt still has +/// its old native versions sitting in `script`. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_superseded_native_version_does_not_count(db: Pool) { + plant_producer(&db, "u/test-user/project", "postgresql", 1).await; + sqlx::query("UPDATE script SET archived = true WHERE hash = 1") + .execute(&db) + .await + .expect("archive the old version"); + plant_producer(&db, "u/test-user/project", "dbt", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +/// The rows of the script being deployed describe the version it replaces, so a +/// script dropping its `// materialize` while adding a subscription would +/// otherwise count itself as the producer that wakes it — and commit a dormant +/// edge. It can never be that producer anyway: the dispatcher skips self-loops. +/// Under a rename that write sits at the OLD path, which the deploy is removing +/// in the same uncommitted transaction, so both paths have to be excluded. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_subscriber_is_never_its_own_producer(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_renamed_producer_is_excluded_too(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer( + &db, + WS, + RELATION, + &[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()] + ) + .await + .unwrap(), + Some("u/test-user/project".to_string()), + "the write this deploy is moving off the old path cannot wake the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_agrees_with_the_singular_one(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, "u/test-user/consumer").await; + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → u/test-user/consumer")], + "dbt alone builds it, so the subscription can never be woken" + ); + + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "a native producer wakes it, so the edge is live" + ); +} + +/// The two ways the set form could stop meaning what the singular one means: a +/// relation nothing produces is deploy order rather than a dormant edge, and a +/// subscriber's own write is not a producer that can wake it — the dispatcher +/// skips self-loops, so that edge is dormant and has to be named. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_matches_on_the_edge_cases_too(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, SUBSCRIBER).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "nothing produces it yet, so nothing is dormant" + ); + + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → {SUBSCRIBER}")], + "the subscriber's own write cannot wake it, so dbt is still the sole producer" + ); +} diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index af9ca957b9..a3f7154cda 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result // A dbt run records the relations it builds, so it looks like a producer // here — but dbt does not trigger downstream runs. Its own DAG is dbt's to // order; the only thing a cascade would add is waking Windmill scripts that - // read a mart, and nothing outside dbt can declare a `dbt://` write, so - // that edge exists in one direction only. Cascading from a project whose - // per-run selection can build any subset of itself needs a per-run write set - // to be correct, which is a design worth doing deliberately rather than - // inferring. Until then dbt materializes and reports; it does not dispatch. + // read a mart. Cascading from a project whose per-run selection can build any + // subset of itself needs a per-run write set to be correct, which is a design + // worth doing deliberately rather than inferring: the deploy-time write set is + // not what ran, and the per-relation state table keeps one row per relation. + // Until then dbt materializes and reports; it does not dispatch. The opposite + // direction does: a native `// materialize manual dbt://…` script reaches the + // fan-out below on the ordinary path, on the strength of its own asset rows. if job.script_lang == Some(ScriptLang::Dbt) { return Ok(DispatchResult::default()); } diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index f9bfe2f4df..cf3c73f8da 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -50,9 +50,9 @@ use windmill_common::{ error::{self, Error, JsonResult, Result}, get_database_url, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only, - fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, WithDraftOverlay, - WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only, fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, + WithDraftOverlay, WithDraftQuery, }, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, @@ -89,6 +89,9 @@ pub fn workspaced_service() -> Router { .route("/git_commit_hash/{*path}", get(get_git_commit_hash)) .route("/type/list", get(list_resource_types)) .route("/type/listnames", get(list_resource_types_names)) + .route("/type/resource_counts", get(list_resource_counts_by_type)) + .route("/type/hub/info", get(list_hub_resource_type_info)) + .route("/type/hub/pick/{name}", post(pick_hub_resource_type)) .route("/type/get/{name}", get(get_resource_type)) .route("/type/exists/{name}", get(exists_resource_type)) .route("/type/update/{name}", post(update_resource_type)) @@ -134,7 +137,10 @@ pub struct EditResourceType { /// `Option` conflates: an absent field leaves the extension alone, while an /// explicit `null` clears it. A hub pull relies on both — a type that stops /// being a file type has to stop being one locally too. - #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] pub format_extension: Option>, } @@ -1309,6 +1315,17 @@ async fn delete_resource( let path = path.to_path(); check_scopes(&authed, || format!("resources:write:{}", path))?; + + // Ahead of the deploy rules: nothing is deployed at a draft-only path, so + // gating this discard on them would strand the row in a protected workspace. + // Ahead of the transaction too — the not-found branch other kinds hang this + // off is the `not_found_if_none` below, past the linked-variable cascade. + if delete_draft_only_for_path(&db, &w_id, UserDraftItemKind::Resource, path, &authed.email) + .await? + { + return Ok(format!("draft-only resource {} deleted", path)); + } + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -1320,6 +1337,7 @@ async fn delete_resource( { return Err(Error::PermissionDenied(msg)); } + let mut tx = user_db.begin(&authed).await?; // Capture resource data for trashbin before deleting @@ -2566,6 +2584,352 @@ async fn list_resource_types_names( Ok(Json(rows)) } +#[derive(Serialize)] +struct ResourceTypeCount { + resource_type: String, + count: i64, +} + +/// How many resources of each type this workspace holds — how popular a type is *here*, +/// which is what the pickers rank on below the hub's own pick counts. +async fn list_resource_counts_by_type( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + // A count per type is aggregate, so there is no path to narrow it by: a token scoped + // to individual resources gets nothing rather than a total spanning paths it cannot + // read. Callers treat the refusal as "no local signal". + check_scopes(&authed, || "resources:read".to_string())?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| ResourceTypeCount { resource_type: r.resource_type, count: r.count }) + .collect(), + )) +} + +/// A hub read, remembered with the hub it came from: `hub_base_url` is a live instance +/// setting, so a cache ignoring it would keep serving the previous hub's answers. +struct HubCached { + hub_base_url: String, + fetched_at: std::time::Instant, + value: T, +} + +/// The index changes only when a resource type is published to the hub; picks move slowly +/// and only reorder a list. Both are read on every drawer open, hence caching at all. +const HUB_RT_INDEX_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60); +const HUB_RT_PICKS_TTL: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// How long a *failed* index read is remembered. Short, and deliberately not the hour a +/// success is good for: this read is on the path of every picker open, so an unreachable +/// hub must not cost an outbound timeout each time, while one blip must not silence pick +/// reporting for an hour. +const HUB_RT_INDEX_FAILURE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +static HUB_RT_INDEX: LazyLock< + std::sync::RwLock>>>>, +> = LazyLock::new(|| std::sync::RwLock::new(None)); +static HUB_RT_PICKS: LazyLock>>>> = + LazyLock::new(|| std::sync::RwLock::new(None)); + +fn hub_cache_get( + cache: &std::sync::RwLock>>, + hub_base_url: &str, + ttl: std::time::Duration, +) -> Option { + let guard = cache.read().ok()?; + let entry = guard.as_ref()?; + (entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl) + .then(|| entry.value.clone()) +} + +fn hub_cache_put(cache: &std::sync::RwLock>>, hub_base_url: &str, value: T) { + if let Ok(mut guard) = cache.write() { + *guard = Some(HubCached { + hub_base_url: hub_base_url.to_string(), + fetched_at: std::time::Instant::now(), + value, + }); + } +} + +#[derive(Deserialize)] +struct HubResourceTypeEntry { + id: i64, + name: String, + /// Optional so a hub that does not send it costs only the mapping. Required, it would + /// fail the whole parse and take pick reporting — which needs just the id — with it. + #[serde(default)] + app: Option, +} + +#[derive(Clone)] +struct HubResourceType { + id: i64, + /// The integration the type belongs to. Usually the type's own name, but not always: + /// `discord_webhook` and `discord_bot_configuration` are both `discord`, and only the + /// hub knows that. Without it a workspace holding a `discord_webhook` resource looks + /// like one that has never touched Discord. + app: String, +} + +/// Reads the index cache, choosing the TTL by what is stored: a failure expires far sooner +/// than a success. `None` is a miss, `Some(None)` a remembered failure. +fn hub_index_cached(hub_base_url: &str) -> Option>> { + let guard = HUB_RT_INDEX.read().ok()?; + let entry = guard.as_ref()?; + if entry.hub_base_url != hub_base_url { + return None; + } + let ttl = if entry.value.is_some() { + HUB_RT_INDEX_TTL + } else { + HUB_RT_INDEX_FAILURE_TTL + }; + (entry.fetched_at.elapsed() < ttl).then(|| entry.value.clone()) +} + +/// What the hub knows about every published resource type, keyed by the name Windmill +/// addresses it by. `None` when the hub cannot be reached or does not answer with a list. +async fn hub_resource_types( + db: &DB, + hub_base_url: &str, +) -> Option> { + if let Some(cached) = hub_index_cached(hub_base_url) { + return cached; + } + let index = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/list"), + false, + None, + Some(db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + // Only the id and the app are kept. That listing carries every type's schema — + // around a megabyte — and neither reporting a pick nor grouping types by + // integration needs it. + Some( + response + .json::>() + .await + .ok()? + .into_iter() + .map(|rt| { + let app = rt.app.unwrap_or_else(|| rt.name.clone()); + (rt.name, HubResourceType { id: rt.id, app }) + }) + .collect::>(), + ) + } + .await; + + hub_cache_put(&HUB_RT_INDEX, hub_base_url, index.clone()); + index +} + +#[derive(Serialize)] +struct PickHubResourceTypeResult { + success: bool, +} + +/// Tells the hub a resource type was taken into a workspace, which is the counter its +/// `/resource_types/picked` ranking reads. +/// +/// Never fails the caller: a hub predating the route, an unreachable one, and a type that +/// is local-only all mean the same thing — not counted — and the request that reaches here +/// has already saved the user's resource. +/// +/// POST, and scoped as a write, because it changes state on the hub under the instance's +/// own credentials. The sibling `/type/*` routes are metadata reads that a `resources:run` +/// app-embed token may make, and both the method and this check keep such a token — which +/// is untrusted app JavaScript — from driving hub counters through us. +async fn pick_hub_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Path((_w_id, name)): Path<(String, String)>, +) -> JsonResult { + check_scopes(&authed, || "resources:write".to_string())?; + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let success = async { + let id = hub_resource_types(&db, &hub_base_url).await?.get(&name)?.id; + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/{id}/pick"), + false, + None, + Some(&db), + ) + .await + .ok()?; + Some(response.status().is_success()) + } + .await + .unwrap_or(false); + + if !success { + tracing::debug!("hub did not record a pick for resource type {name}"); + } + Ok(Json(PickHubResourceTypeResult { success })) +} + +#[derive(Deserialize, Clone)] +struct HubResourceTypePicks { + name: String, + /// The hub counts picks in a bigint, which its driver serialises as a string. + #[serde(deserialize_with = "windmill_common::more_serde::maybe_number")] + picks: i64, +} + +#[derive(Deserialize)] +struct HubPickedResourceTypes { + resource_types: Vec, +} + +#[cfg(test)] +mod hub_picks_tests { + use super::*; + + /// Two properties of the index cache that a later edit could quietly drop: it is keyed on + /// the hub it came from, and a failure is forgotten long before a success is. + #[test] + fn the_index_cache_is_keyed_on_the_hub_and_forgets_failures_sooner() { + let index = || { + Some(HashMap::from([( + "slack".to_string(), + HubResourceType { id: 1, app: "slack".to_string() }, + )])) + }; + + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", index()); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_some())); + // Switching hubs must miss rather than serve the previous hub's mapping. + assert!(hub_index_cached("https://other.example").is_none()); + + // A remembered failure reads as a hit (so the hub is not re-attempted) carrying + // nothing, and only until the shorter of the two TTLs. + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", None); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_none())); + assert!(HUB_RT_INDEX_FAILURE_TTL < HUB_RT_INDEX_TTL); + + if let Ok(mut guard) = HUB_RT_INDEX.write() { + *guard = None; + } + } + + /// The hub counts picks in a bigint, which postgres.js serialises as a string. Typing + /// the field as a plain i64 fails the whole response, and the ranking silently empties. + #[test] + fn picks_decode_from_a_string_or_a_number() { + let parsed: HubPickedResourceTypes = serde_json::from_str( + r#"{"resource_types":[{"name":"slack","picks":"42"},{"name":"github","picks":7}]}"#, + ) + .unwrap(); + assert_eq!( + parsed + .resource_types + .iter() + .map(|rt| (rt.name.as_str(), rt.picks)) + .collect::>(), + vec![("slack", 42), ("github", 7)] + ); + } +} + +/// One hub resource type as the pickers need it. +#[derive(Serialize)] +struct HubResourceTypeInfo { + name: String, + /// The integration it belongs to, so a caller can total a workspace's resources per + /// integration rather than per type. + app: String, + picks: i64, +} + +/// What the hub knows about its resource types: which integration each belongs to, and how +/// often each has been picked. +/// +/// Empty rather than an error when the hub answers neither read, so the pickers treat an +/// older or private hub as "no hub signal" and fall back to what the workspace itself uses. +/// The two reads degrade independently: a hub that lists types but has no `picked` route +/// still supplies the type-to-integration mapping, which is what decides whether a +/// workspace's resources are recognised as belonging to an integration at all. +async fn list_hub_resource_type_info( + Extension(db): Extension, +) -> JsonResult> { + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let picks = match hub_cache_get(&HUB_RT_PICKS, &hub_base_url, HUB_RT_PICKS_TTL) { + Some(picks) => picks, + None => { + let fetched = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/picked"), + false, + Some(vec![("limit", "200".to_string())]), + Some(&db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + Some( + response + .json::() + .await + .ok()? + .resource_types, + ) + } + .await + .unwrap_or_default(); + hub_cache_put(&HUB_RT_PICKS, &hub_base_url, fetched.clone()); + fetched + } + }; + + let mut picks_by_name: HashMap = + picks.into_iter().map(|rt| (rt.name, rt.picks)).collect(); + let index = hub_resource_types(&db, &hub_base_url) + .await + .unwrap_or_default(); + + let mut info: Vec = index + .into_iter() + .map(|(name, rt)| HubResourceTypeInfo { + picks: picks_by_name.remove(&name).unwrap_or(0), + name, + app: rt.app, + }) + .collect(); + // What the index did not account for is a type the picks read knows and the listing does + // not — which is what a hub answering only the second read looks like. Its own name is + // the same guess a caller makes for any type the mapping misses. + info.extend( + picks_by_name + .into_iter() + .map(|(name, picks)| HubResourceTypeInfo { app: name.clone(), name, picks }), + ); + + Ok(Json(info)) +} + async fn get_resource_type( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 8ef7c1ee39..953f3a571b 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -18,8 +18,9 @@ use windmill_common::{ error::{Error, JsonResult, Result}, trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource}, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows, - overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, + WithDraftQuery, }, utils::{paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, @@ -990,6 +991,18 @@ async fn delete_trigger( .await?; if !deleted { + drop(tx); + if delete_draft_only_for_path( + &db, + &workspace_id, + T::user_draft_item_kind(), + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only trigger '{}' deleted", path)); + } return Err(Error::NotFound(format!( "Trigger not found at path: {}", path diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index 463a365b06..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -14,17 +14,20 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, where `` is the name the - /// workspace configures it under. + /// A warehouse relation, `dbt:////`, where + /// `` is the name the workspace configures it under. /// - /// The SCHEME names the producer — dbt is the only thing that creates one — - /// while the PATH stays the physical relation, because that is what two - /// projects agree on: a mart one builds is a `source` the next reads, and - /// their dbt `unique_id`s differ (`model.a.orders` vs - /// `source.b.analytics.orders`) where the relation does not - /// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that - /// reader — the shared node is lineage, not a cascade edge. + /// The SCHEME names the namespace dbt made rather than an exclusive + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, + /// because that is what two producers agree on: a mart one builds is a + /// `source` the next reads, and their dbt `unique_id`s differ + /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does + /// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the + /// readers of what it built — that shared node is lineage, not a cascade + /// edge — while a declared write does (decision 25). Dbt, } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index e75afc053e..5e95a1e20a 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -114,6 +114,10 @@ rust_decimal.workspace = true jsonwebtoken.workspace = true sha2.workspace = true hmac.workspace = true +# Reads the dbt engine's `target/index/*.parquet`, which is where column-level +# lineage lives. Unconditional rather than behind the `parquet` FEATURE: that one +# is object storage, and a build without it still runs dbt jobs. +parquet.workspace = true pem = { workspace = true, optional = true } rsa = { workspace = true, optional = true } urlencoding.workspace = true diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index b3324f87ba..b577097c9d 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -408,16 +408,6 @@ pub fn create_empty_dir(path: &PathBuf) -> std::io::Result<()> { } } -/// Signals a detached `spawn_blocking` task that the future awaiting it is -/// gone, so it can stop instead of running to completion in the background. -struct AbortOnDrop(std::sync::Arc); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.store(true, std::sync::atomic::Ordering::Relaxed); - } -} - /// Lay down the tree of an app-backed repository, which git can't clone /// because its URL carries no credential. /// @@ -490,7 +480,7 @@ async fn fetch_repo_archive( // stopping it, so the flag is what a cancelled job uses to reach the // extraction loop. The guard sets it when this future is dropped. let aborted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _abort_on_drop = AbortOnDrop(aborted.clone()); + let _abort_on_drop = crate::common::AbortOnDrop(aborted.clone()); let unpack_archive = download_archive.clone(); tokio::task::spawn_blocking(move || { unpack_repo_archive(&unpack_archive, &download_target, &aborted) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index f6391eab07..0a5deb52d2 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -67,6 +67,20 @@ mount { #[cfg(not(debug_assertions))] pub const DEV_CONF_NSJAIL: &str = ""; +/// Tells a `spawn_blocking` task to stop when the future awaiting it goes away. +/// +/// Dropping a `JoinHandle` detaches the task rather than cancelling it, so a +/// cancelled or timed-out phase otherwise leaves the blocking pool working on an +/// answer nobody will read. Hold one of these beside the handle and have the +/// blocking loop check the flag. +pub(crate) struct AbortOnDrop(pub(crate) std::sync::Arc); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.store(true, std::sync::atomic::Ordering::Relaxed); + } +} + /// Turn a JSON value into the string a shell/CLI arg should receive: a JSON string /// becomes its inner value, anything else is re-serialized compactly. pub(crate) fn raw_to_string(x: &str) -> String { diff --git a/backend/windmill-worker/src/dbt_column_index.rs b/backend/windmill-worker/src/dbt_column_index.rs new file mode 100644 index 0000000000..ab5853b9b6 --- /dev/null +++ b/backend/windmill-worker/src/dbt_column_index.rs @@ -0,0 +1,589 @@ +//! Column-level lineage and real column schemas, from the engine's own static +//! analysis. +//! +//! `manifest.json` carries neither. What does is the parquet index an engine +//! writes under `dbt compile --static-analysis strict --write-index`: +//! `dbt.column_lineage.parquet` (column-to-column edges, each labelled `copy`, +//! `mod` or `scan`) and `dbt.node_columns.parquet` (every column of every node, +//! typed and ordered, rather than only the ones an author documented). +//! +//! Four properties shape everything here, all of them measured against the real +//! engines rather than assumed: +//! +//! - **Strict analysis rejects SQL the default accepts.** An unresolvable +//! identifier is an error under `strict` and compiles fine otherwise, so this +//! is a SEPARATE pass with its own `--target-path`, never a flag on the build, +//! and it is opt-in per project. +//! - **A failed pass still writes the index**, holding every edge of the models +//! that did analyze. So the artifact is read whatever the exit status. +//! - **The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 accepts +//! `--write-index`, declares the views over these two tables in its own +//! `views.sql`, and writes neither file; only Fusion does today. Nothing here +//! asks which engine it is beyond "has the flag" — a release that starts +//! writing them is picked up with no change. +//! - **An incremental model has two shapes, and one ingest holds one of them.** +//! `is_incremental()` is false when the target does not exist or the build +//! is `--full-refresh`, so the `{{ this }}` self-join — and any `ref()` inside +//! that branch — compiles only in the other case. What this stores is +//! therefore what the compile in front of it saw: at DEPLOY, before the first +//! build, that is the cold shape, and a project deployed again after its +//! tables exist stores the incremental one for the same source. Nothing here +//! can reconcile that; dbt has no mode that emits both. The flag is taken from +//! the build so a per-run ingest matches its own run, and the version's graph +//! is honest about the compile that produced it rather than about every run +//! that will follow. + +use std::collections::HashSet; +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::path::Path; +use std::time::Duration; + +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::record::{Field, Row}; +use uuid::Uuid; +use windmill_common::dbt_manifest::{ + is_direct, ColumnIndex, IndexedColumn, IngestedColumnEdge, MAX_COLUMN_EDGES, +}; +use windmill_common::error; +use windmill_common::worker::Connection; +use windmill_parser_yaml::dbt::DbtDescriptor; +use windmill_queue::append_logs; + +use crate::dbt_executor::{dbt_command, Invocation, PreparedProject}; +use crate::handle_child::JobCtx; + +/// Where the lineage pass writes, relative to the project directory. +/// +/// Its own tree, not the runtime's `wm_target`: a `dbt compile` writes +/// `manifest.json` and `run_results.json` like any other invocation, and after a +/// build those two are what the graph ingest and `dbt retry` read. +const CLL_ARTIFACTS_DIR: &str = "wm_target_cll"; + +const COLUMN_LINEAGE_PARQUET: &str = "dbt.column_lineage.parquet"; +const NODE_COLUMNS_PARQUET: &str = "dbt.node_columns.parquet"; + +/// Run the lineage pass and read what it produced. +/// +/// Two steps with deliberately different contracts, because conflating them is +/// what made a best-effort annotation able to fail the job it annotates: +/// +/// - [`compile_index`] runs a subprocess and owns the JOB's semantics. Only a +/// cancellation or the job's own deadline can `Err` out of it; a non-zero exit +/// and an over-long output are outcomes, not failures. +/// - [`read_index`] owns the ARTIFACT's semantics. Reading it never fails the +/// job on the artifact's account: an absent, unreadable or partial index is a +/// value, not an error. It runs UNDER the poller all the same, so the job can +/// still end the phase — a cancel, a completion or the phase timeout — which +/// is the job's semantics reaching in, not the artifact's reaching out. +/// +/// The phase budget wraps the compile alone, because it exists to leave the +/// BUILD its share of the clock and only the compile can spend that share +/// unboundedly. The decode's own end is the job's: the poller it runs under +/// stops it when the job stops. +pub(crate) async fn collect( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + // The dbt subcommand the job runs, which decides the effective + // `--full-refresh` — see `dbt_executor::full_refresh`. + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + kept: &HashSet<&str>, +) -> error::Result> { + if !descriptor.column_lineage { + return Ok(None); + } + if !p.engine.engine.writes_column_index() { + append_logs( + job_id, + w_id, + format!( + "\n`column_lineage` is set, but the {} engine has no `--write-index`: column \ + lineage needs an engine that does static analysis. The rest of the graph is \ + unaffected.\n", + p.engine.engine.as_str() + ), + conn, + ) + .await; + return Ok(None); + } + + let index_dir = p.project_dir.join(CLL_ARTIFACTS_DIR).join("index"); + let Some(compiled) = compile_index(p, descriptor, inv, command, ctx, job_id, w_id, conn).await? else { + return Ok(None); + }; + let coverage = Coverage::of(&compiled); + + // Decoded UNDER the poller, not followed by a check of its own. The decode is + // the one phase of this pass with no subprocess behind it, so nothing else + // heartbeats while it runs: left alone, a large index is a silent worker for + // as long as it takes, which the zombie sweep reads as a dead job and + // restarts. The poller pings throughout and ends this with an `Err` if the + // job is cancelled or completed meanwhile — the job's own semantics, which + // this module may always propagate. + let artifact = crate::handle_child::run_future_with_polling_update_job_poller( + *job_id, + ctx.timeout(), + conn, + ctx.mem_peak, + ctx.canceled_by, + async { Ok(read_index(&index_dir, kept).await) }, + ctx.worker_name, + w_id, + &mut Some(ctx.occupancy_metrics), + Box::pin(futures::stream::empty()), + ) + .await?; + + // What only the pass knows. The COUNTS are logged where the index is folded + // into the graph, since the graph decides how much of it is kept. + let note = match artifact { + Artifact::Read(index) => { + if let Some(note) = coverage.caveat() { + log(job_id, w_id, note, &compiled.stderr, conn).await; + } + return Ok(Some(index)); + } + // The truncated arms come first: a compile stopped part-way explains an + // absent or unreadable artifact, and blaming the engine's capability + // for it sends the reader to check the wrong thing entirely. + Artifact::Missing if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass printed more than this runtime reads and was \ + stopped before it wrote `{COLUMN_LINEAGE_PARQUET}`." + ), + Artifact::Unreadable(why) if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass was stopped for printing more than this \ + runtime reads, and the `{COLUMN_LINEAGE_PARQUET}` it had written could not be read \ + ({why})." + ), + // Said apart from the one below, because it sends the reader somewhere + // else entirely: the engine did its job and this runtime could not read + // what it wrote. + Artifact::Unreadable(why) => format!( + "No column lineage: `{COLUMN_LINEAGE_PARQUET}` was written but could not be read \ + ({why}). The graph is unaffected." + ), + Artifact::Missing => format!( + "No column lineage: the analysis pass wrote no `{COLUMN_LINEAGE_PARQUET}`. Only an \ + engine that computes it does, and only for the warehouses it analyzes natively — \ + the flag alone is not the capability." + ), + }; + // The engine's own diagnostics come along. They are how a reader learns that + // this adapter turned static analysis off, which it reports as a warning on + // a SUCCESSFUL compile that nothing else would show. + log(job_id, w_id, ¬e, &compiled.stderr, conn).await; + Ok(None) +} + +async fn log(job_id: &Uuid, w_id: &str, note: &str, stderr: &str, conn: &Connection) { + append_logs( + job_id, + w_id, + format!("\n{note}\n{}", diagnostics(stderr)), + conn, + ) + .await; +} + +/// How completely the analysis compile covered the project. +/// +/// Every way the COMPILE can disappoint is a value here rather than an error. An +/// `Err` from `compile_index` is the JOB's — a cancellation or its deadline — +/// and must fail it; the pass giving up on its own terms is `Ok(None)` and has +/// already been logged. +enum Coverage { + /// Every model analyzed. + Whole, + /// `--static-analysis strict` rejected part of the project. Whatever it did + /// analyze is still in the index. + Partial, + /// The output ceiling killed the compile mid-run. Distinct from `Partial`: + /// nothing rejected the project, but the index is however far it had got, so + /// it is not `Whole` either. + Truncated, +} + +impl Coverage { + fn of(c: &crate::dbt_executor::Captured) -> Self { + match (c.truncated, c.success) { + (true, _) => Coverage::Truncated, + (false, true) => Coverage::Whole, + (false, false) => Coverage::Partial, + } + } + + /// What to tell the reader when an index WAS produced. `None` for a run that + /// covered everything, which needs no caveat. + fn caveat(&self) -> Option<&'static str> { + match self { + Coverage::Whole => None, + Coverage::Partial => Some( + "Column lineage: `--static-analysis strict` rejected part of the project, so \ + the lineage covers only the models it could analyze.", + ), + Coverage::Truncated => Some( + "Column lineage: the analysis pass printed more than this runtime reads and was \ + stopped, so the lineage covers only the models it had reached.", + ), + } + } +} + +/// Run `dbt compile --static-analysis strict --write-index`, under this phase's +/// share of the job's clock. +/// +/// `Ok(None)` is "the pass gave up and said so"; `Err` is the job's own +/// cancellation or deadline and must propagate. Nothing outlives this function: +/// the budget is a race around the child, and dropping that future kills it +/// through `run_captured`'s `kill_on_drop`. +async fn compile_index( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result> { + // A previous pass in the same job directory — a retry's second attempt — + // would otherwise be read back as this one's answer. + tokio::fs::remove_dir_all(p.project_dir.join(CLL_ARTIFACTS_DIR)) + .await + .ok(); + + let mut cmd = dbt_command( + p, + &[ + "compile", + "--static-analysis", + "strict", + "--write-index", + // Documented as what builds the CLL graph, and `--write-index` alone + // happens to imply it on the engine probed. Passed explicitly so the + // pass does not depend on which of the two is doing the work. + "--write-lineage", + "--target-path", + CLL_ARTIFACTS_DIR, + ], + ); + // The flag already wins over the env var dbt_command sets, but setting both + // means this pass cannot write into the runtime's artifacts even if that + // precedence ever changes — and what is in there after a build is the + // `run_results.json` a `dbt retry` resumes from. + cmd.env("DBT_TARGET_PATH", CLL_ARTIFACTS_DIR); + crate::dbt_executor::add_vars(&mut cmd, descriptor, inv)?; + // The BUILD's answer, not the descriptor's default: `is_incremental()` + // branches on it, so a model reading `{{ this }}` compiles its self-join — + // and any `ref()` inside that branch — only when this is absent. Guessing + // here stores lineage for SQL the run never executed. + if crate::dbt_executor::full_refresh(descriptor, inv, command)? { + cmd.arg("--full-refresh"); + } + // Captured rather than streamed: a strict-analysis failure is a wall of + // diagnostics about SQL the build itself accepts, and this pass decides + // nothing about whether that build runs. + // Read before the future below borrows `ctx` mutably. + let budget = phase_budget(ctx); + let run = crate::dbt_executor::run_captured( + cmd, + "dbt compile (column lineage)", + ctx, + job_id, + w_id, + conn, + CLL_MAX_OUTPUT_BYTES, + // The ceiling is this pass's, not the job's: a compile that prints more + // than it than we care to read has still analyzed the project, and the + // index it wrote is on disk either way. + crate::dbt_executor::Overflow::Truncate, + ); + let Some(budget) = budget else { + return Ok(Some(run.await?)); + }; + match tokio::time::timeout(budget, run).await { + Ok(r) => Ok(Some(r?)), + Err(_) => { + append_logs( + job_id, + w_id, + format!( + "\nNo column lineage: the analysis pass did not finish within {}s, half of \ + what was left of this job's time. The build below gets the rest.\n", + budget.as_secs() + ), + conn, + ) + .await; + Ok(None) + } + } +} + +/// stdout the pass may produce. It is a compile, so this is diagnostics rather +/// than data. +const CLL_MAX_OUTPUT_BYTES: usize = 1 << 20; + +/// The share of the job's remaining wall clock this pass may spend. +/// +/// A per-run refresh ingests BEFORE the build and shares the job's one deadline, +/// so an unbounded pass on a slow project would hand `dbt build` an expired +/// budget and fail the run it exists only to annotate. Half leaves the build at +/// least as long as the annotation was allowed to take. +/// +/// Spent as a race around the COMPILE rather than as a shortened deadline handed +/// to the runner: the runner reports its expiry as an `Err`, indistinguishable +/// from a cancellation or the job's own deadline, and those two MUST fail the +/// job. Expiring here is this budget and nothing else. The child dies with the +/// dropped future through `run_captured`'s `kill_on_drop`; the decode is outside +/// this race and answers to the poller instead. +fn phase_budget(ctx: &JobCtx<'_>) -> Option { + ctx.timeout() + .map(|left| Duration::from_secs((left.max(0) as u64 / 2).max(1))) +} + +/// The tail of what the engine said, bounded. The whole of it is every rendered +/// model on a large project, which is not what a job log is for. +const DIAGNOSTIC_LINES: usize = 40; + +fn diagnostics(out: &str) -> String { + let lines: Vec<&str> = out.lines().filter(|l| !l.trim().is_empty()).collect(); + let tail = &lines[lines.len().saturating_sub(DIAGNOSTIC_LINES)..]; + match tail.is_empty() { + true => String::new(), + false => format!("{}\n", tail.join("\n")), + } +} + +/// What came back from the artifact. Never an `Err`: nothing the file does or +/// fails to do is a reason to fail a job. `Unreadable` is separate from +/// `Missing` because the two send a reader looking in different places — one at +/// their engine and adapter, the other at a file that exists. +enum Artifact { + Read(ColumnIndex), + Missing, + Unreadable(String), +} + +/// Read both parquets, if the lineage one is there. +/// +/// The column schemas alone are not worth a graph: they arrive with the lineage +/// or not at all, and a node's declared columns already answer for the case +/// where the pass never ran. +async fn read_index(index_dir: &Path, kept: &HashSet<&str>) -> Artifact { + let lineage = index_dir.join(COLUMN_LINEAGE_PARQUET); + if !tokio::fs::try_exists(&lineage).await.unwrap_or(false) { + return Artifact::Missing; + } + let columns = index_dir.join(NODE_COLUMNS_PARQUET); + // Owned, because the decode moves to a blocking thread. The index describes + // the whole project while this graph describes one selection of it, so + // scoping HERE is what keeps the bound below from being spent on rows the + // graph would discard anyway. + let kept: HashSet = kept.iter().map(|s| (*s).to_string()).collect(); + // Dropping the handle of a blocking task does NOT stop it: the poller + // cancelling this phase would otherwise leave a thread decoding millions of + // rows for a job that is over. `abandoned` is set when this future is + // dropped, and the row loop reads it. + let abandoned = Arc::new(AtomicBool::new(false)); + let _stop = crate::common::AbortOnDrop(abandoned.clone()); + // Decompressing and decoding a parquet is CPU work on a file the engine just + // wrote, so it does not belong on the runtime's poll thread. + let read = tokio::task::spawn_blocking(move || { + read_index_blocking(&lineage, &columns, &kept, &abandoned) + }) + .await; + match read { + Ok(Ok(index)) => Artifact::Read(index), + Ok(Err(e)) => Artifact::Unreadable(e.to_string()), + Err(e) => Artifact::Unreadable(e.to_string()), + } +} + +fn read_index_blocking( + lineage: &Path, + columns: &Path, + kept: &HashSet, + abandoned: &AtomicBool, +) -> error::Result { + let mut out = ColumnIndex::default(); + // ONE pass, with the two kinds bucketed as they arrive. `copy` and `mod` say + // the value itself travelled, so they get the whole budget; `scan` — the + // column was read to produce the ROW, which reaches every output column of + // its model and is the bulk of a wide project's index — fills only what is + // left over at the end. Reading the file twice to get that ordering would + // double the decode of exactly the large index this bound exists for. + let mut scan: Vec = Vec::new(); + for_each_row(lineage, abandoned, |row| { + let lineage_kind = string(row, "lineage_kind"); + let parent_unique_id = string(row, "from_node_unique_id"); + let child_unique_id = string(row, "to_node_unique_id"); + let parent_column = string(row, "from_column_name"); + let child_column = string(row, "to_column_name"); + // A column of a node the analysis could not name is not an endpoint the + // graph can draw, and neither is one outside this graph's nodes. + if parent_column.is_empty() + || child_column.is_empty() + || !kept.contains(&parent_unique_id) + || !kept.contains(&child_unique_id) + { + return ControlFlow::Continue(()); + } + let edge = IngestedColumnEdge { + parent_unique_id, + parent_column, + child_unique_id, + child_column, + lineage_kind, + }; + // The bound covers BOTH buckets, so the pass never holds more than one + // budget's worth however the kinds are distributed. + let held = out.edges.len() + scan.len(); + if is_direct(&edge.lineage_kind) { + // A direct edge displaces a `scan` one: the budget is spent on + // value flow first. + if held >= MAX_COLUMN_EDGES { + scan.pop(); + } + out.edges.push(edge); + // The edge that FILLS the budget ends the read, not the next one to + // arrive: once the displacing kind is full nothing later in the file + // can be kept, and waiting for another direct edge to say so decodes + // a `scan`-only tail all the way to the backstop for nothing. + return match out.edges.len() >= MAX_COLUMN_EDGES { + true => ControlFlow::Break(()), + false => ControlFlow::Continue(()), + }; + } + if held < MAX_COLUMN_EDGES { + scan.push(edge); + } + // Not a stopping point even when full: a direct edge still to come takes + // a `scan` entry's place. + ControlFlow::Continue(()) + })?; + out.edges.append(&mut scan); + // Absent is normal — an engine can write the lineage table and not this one — + // and unreadable is not worth losing the lineage over. + let mut held = 0usize; + let _ = for_each_row(columns, abandoned, |row| { + let unique_id = string(row, "unique_id"); + let name = string(row, "column_name"); + if held >= MAX_INDEXED_COLUMNS { + return ControlFlow::Break(()); + } + if name.is_empty() || !kept.contains(&unique_id) { + return ControlFlow::Continue(()); + } + held += 1; + // The author's `data_type` where `schema.yml` gives one, since that is + // what the project calls the column; the analysis's own inference + // otherwise. + let column_type = match string(row, "declared_type") { + t if !t.is_empty() => t, + _ => string(row, "inferred_type"), + }; + out.columns + .entry(unique_id) + .or_default() + .push(IndexedColumn { + name, + column_type, + index: int(row, "column_index").unwrap_or(i64::MAX), + }); + ControlFlow::Continue(()) + }); + Ok(out) +} + +/// The most rows of `dbt.node_columns.parquet` one pass keeps. One per column of +/// the project, so the same bound as the edges is far more than any project +/// reaches; it exists for the same reason. +const MAX_INDEXED_COLUMNS: usize = MAX_COLUMN_EDGES; + +/// The most rows of an index one pass DECODES, whatever it keeps of them. +/// +/// A bound on work rather than on memory, and the two are separate because the +/// input this defends against is the one that cannot be collected: `scan` +/// lineage is emitted from every predicate and join column to every output +/// column, so a project shaped that way writes an index whose row count is +/// quadratic in its widest model. This pass runs outside the phase budget, on a +/// blocking thread, and nothing the file contains may fail a deploy or a run — +/// so the file it walks needs an end even when almost nothing in it is +/// retained. The abandonment flag ends it sooner when the job is over; this is +/// the bound for a job that is not. +const MAX_INDEX_ROWS: usize = 4_000_000; + +/// Decode a parquet a row at a time, handing each to `f` and never holding two. +/// +/// Collecting first would put a `Vec` — each row carrying its own copy of +/// every column NAME — in front of the caller's own bound, which is what would +/// take the worker process down on the index described above. +/// +/// `f` says when it has all it will take, and that is the ordinary end: this +/// runs outside the phase budget, so every row decoded past the point of being +/// able to keep one is wall clock the build below does not get. +fn for_each_row( + path: &Path, + abandoned: &AtomicBool, + mut f: impl FnMut(&Row) -> ControlFlow<()>, +) -> error::Result<()> { + let fail = |e: parquet::errors::ParquetError| { + error::Error::internal_err(format!("reading {}: {e}", path.display())) + }; + let file = std::fs::File::open(path) + .map_err(|e| error::Error::internal_err(format!("opening {}: {e}", path.display())))?; + let reader = SerializedFileReader::new(file).map_err(fail)?; + for (n, row) in reader.get_row_iter(None).map_err(fail)?.enumerate() { + // Nobody is waiting for this any more — the job was cancelled, completed + // or ran out of time while it decoded. + if abandoned.load(Ordering::Relaxed) { + break; + } + if n >= MAX_INDEX_ROWS { + tracing::warn!( + "dbt column index: {} holds more than {MAX_INDEX_ROWS} rows; the rest is dropped", + path.display() + ); + break; + } + if f(&row.map_err(fail)?).is_break() { + break; + } + } + Ok(()) +} + +/// By NAME, not by position: these tables are the engine's own schema and it +/// adds columns to them between releases. +fn field<'a>(row: &'a Row, name: &str) -> Option<&'a Field> { + row.get_column_iter() + .find(|(k, _)| k.as_str() == name) + .map(|(_, v)| v) +} + +fn string(row: &Row, name: &str) -> String { + match field(row, name) { + Some(Field::Str(s)) => s.clone(), + Some(Field::Bytes(b)) => String::from_utf8_lossy(b.data()).into_owned(), + _ => String::new(), + } +} + +fn int(row: &Row, name: &str) -> Option { + match field(row, name) { + Some(Field::Long(v)) => Some(*v), + Some(Field::Int(v)) => Some(*v as i64), + Some(Field::Short(v)) => Some(*v as i64), + Some(Field::UInt(v)) => Some(*v as i64), + Some(Field::ULong(v)) => i64::try_from(*v).ok(), + _ => None, + } +} diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 33ddd6c32e..bb449f1810 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -19,12 +19,13 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::client::AuthedClient; use windmill_common::error::{self, Error}; +use windmill_common::jobs::JobKind; use windmill_common::materialization::{ record_materialization, MaterializationStatus, RecordMaterializationRequest, }; use windmill_common::worker::{to_raw_value, write_file, Connection}; use windmill_parser_yaml::{ - parse_dbt_descriptor, DbtDescriptor, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, + parse_dbt_descriptor, DbtDescriptor, DbtEngine, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, DBT_COMMAND_LABEL, DBT_DEFAULT_WAREHOUSE, }; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -37,6 +38,9 @@ use crate::dbt_engine::{provision_engine, ProvisionedEngine, DBT_CACHE_DIR}; use crate::dbt_profiles::{ ensure_adapter_licensed, render_dbt_profile, render_profile, DbtAdapter, KnownAdapter, }; +use crate::dbt_state::{ + environment_label, prepare_deferral, write_state_dir, Deferral, StateManifest, STATE_DIR, +}; use crate::handle_child::{ get_mem_peak, handle_child, run_future_with_polling_update_job_poller, JobCtx, JobDeadline, }; @@ -121,6 +125,12 @@ pub struct DbtRunResult { /// the same project — cannot get them from the job. #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub invocation_args: std::collections::HashMap>, + /// The run whose stored state this one resolved its unbuilt `ref()`s + /// through, absent when it deferred to none. What a deferring run built + /// against is otherwise unrecoverable: the state is replaced by the next + /// successful run of that environment. + #[serde(skip_serializing_if = "Option::is_none")] + pub deferred_to: Option, } #[derive(Serialize, Debug, Default)] @@ -189,7 +199,13 @@ pub(crate) async fn handle_dbt_job( // result publishes, and both describe an invocation of this script, not one // executor's view of it. let raw_args = job.args.as_ref().map(|a| a.0.clone()).unwrap_or_default(); - let inv = Invocation { args: args.clone(), raw_args, envs: envs.clone(), strict: true }; + let inv = Invocation { + args: args.clone(), + raw_args, + envs: envs.clone(), + deferral: None, + strict: true, + }; // One wall clock for the whole job. A dbt job is a sequence of // subprocesses — provision, deps, parse, ls, build, then the // `after_all` tests — and each would otherwise resolve the job's full @@ -262,6 +278,16 @@ pub(crate) async fn handle_dbt_job( // applies — nothing is built, so there is no test phase, no materialization, // no retry state and no ownership to publish. if command == "parse" { + // Checked here rather than at the seam below, which cannot tell a parse + // from a run that simply left `defer` off: a parse never reaches the + // deferral at all, so it is the one caller for which "turn `defer` on" + // would be advice that leads nowhere. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + StateAccess::Never(&command), + !selection_is_overridden(&descriptor, &inv.args)?, + )?; return run_parse_only( &prepared, &descriptor, @@ -364,6 +390,84 @@ pub(crate) async fn handle_dbt_job( inv }; + // Read AFTER the retry restore, so a retry defers exactly as the run it + // resumes did: a retry's own arguments are the command block alone, and the + // relations its unbuilt `ref()`s resolve to must not depend on that. + let defer = arg_bool(&inv.args, "defer")?.unwrap_or(descriptor.defer); + // Before the state is fetched, not only at the seam where the selection + // reaches dbt: a selector that cannot work whatever the state says would + // otherwise be masked by the "nothing published yet" refusal, which sends the + // caller to publish a state that will not help. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + if defer { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(&descriptor, &inv.args)?, + )?; + // A `show` defers too, and every engine takes the flags on it: it COMPILES + // the model it previews, so a model whose upstream this environment built and + // this run did not is exactly the case a deferral exists for. + let inv = if defer { + // Refused before anything runs. `dbt retry` reads the run it resumes + // from `--state`, the flag a deferral needs, so an engine without + // `--defer-state` can be given one or the other: told to defer, it + // resumes the stored state's own (successful) results and rebuilds + // nothing, and left alone it rebuilds the failed nodes with every + // `ref()` resolving into the schema THIS run writes — which for the + // narrowed run a deferral exists to serve is not where those models go. + if command == "retry" && !prepared.engine.engine.has_defer_state_flag() { + return Err(Error::BadRequest(format!( + "`{}` cannot resume a run that deferred: `dbt retry` takes the run it resumes \ + from `--state`, which is also where a deferral reads its manifest, and this \ + engine has no `--defer-state` to tell the two apart. Run the script again \ + instead of resuming it, or move the project to dbt-core-1x", + prepared.engine.engine.as_str() + ))); + } + let deferral = prepare_deferral(&prepared, &job.workspace_id, job_dir, conn).await?; + // Only answerable once the state is loaded: `defer` is enough for a + // `state:` method, which reads the manifest every publication carries, + // but a `result:` one reads `run_results.json` — and a build recovered by + // node retry publishes without it, since the results it holds describe + // only the nodes the retry rebuilt. dbt-core then raises an INTERNAL + // error and the Rust engines match nothing and exit 0. + if !deferral.has_run_results + && selection_names( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + &["result"], + ) + { + return Err(Error::BadRequest(format!( + "a `result:` selector reads `run_results.json` out of the published state, and \ + the state for this environment ({}) carries only the manifest run {} \ + published: a build recovered by node retry stores none, its results describing \ + the retried nodes rather than the whole build. Run this script once without \ + `defer` and without overrides to publish a complete state, or drop the selector", + environment_label(&prepared), + deferral.published_by + ))); + } + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nDeferring unbuilt refs to the dbt state published by run {}; this run \ + publishes none of its own\n", + deferral.published_by + ), + conn, + ) + .await; + Invocation { deferral: Some(deferral), ..inv } + } else { + inv + }; + // Ingested BEFORE the build, from a `dbt parse` with this run's vars, so the // models shown are the ones about to be built. Rows are keyed by path, version // AND job so no two runs collide; the path-keyed `asset` usage belongs to one @@ -387,7 +491,7 @@ pub(crate) async fn handle_dbt_job( // For a retry the restored manifest already describes the invocation // being resumed, so only the ingest runs — with that invocation's // arguments, which the selection resolver needs to interpolate. - ingest_from_run(&prepared, &descriptor, &inv, &mut ctx, job, conn).await?; + ingest_from_run(&prepared, &descriptor, &inv, &command, &mut ctx, job, conn).await?; } // A read-only command prints rows to stdout, so it is captured rather than @@ -428,9 +532,28 @@ pub(crate) async fn handle_dbt_job( // previous attempt's `run_results.json` is still in the job directory. Never on // an agent worker, which cannot read `v2_job_queue` — the wait below would be // uninterruptible, so a cancelled job would hold its slot and then start dbt. + // And never where the engine cannot be told to defer on a `retry`: the + // rebuild would resolve this run's unbuilt refs into the schema it writes + // into, so the nodes it "recovered" would read from the wrong relations. + // Said out loud below rather than silently skipped. + let retry_would_lose_the_deferral = + inv.deferral.is_some() && !prepared.engine.engine.has_defer_state_flag(); let node_retry = descriptor .retry_failed_nodes - .filter(|_| matches!(conn, Connection::Sql(_))); + .filter(|_| matches!(conn, Connection::Sql(_))) + .filter(|_| !retry_would_lose_the_deferral); + if descriptor.retry_failed_nodes.is_some() && retry_would_lose_the_deferral { + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nSkipping the automatic node retry: `{}` cannot defer on a `dbt retry`\n", + prepared.engine.engine.as_str() + ), + conn, + ) + .await; + } let mut retries_left = node_retry.map(|p| p.attempts()).unwrap_or(0); if let Some(policy) = node_retry.filter(|_| run.is_err()) { retry_failed_nodes( @@ -519,6 +642,56 @@ pub(crate) async fn handle_dbt_job( { tracing::warn!("dbt: could not save retry state for job {}: {e:#}", job.id); } + // What a later run defers to, published by the runs whose relations are the + // SCRIPT's — the same condition that decides whether a run's graph becomes + // what the script owns, and for the same reason: an invocation that scoped + // its own models has no standing to say where this project's relations live. + // Success is the other half, because a relation a deferral resolves to has + // to exist. A `retry` is excluded: its `run_results.json` names only the + // nodes it redid, so publishing it would leave the environment claiming a + // run of a handful of models. + // + // And never a run that DEFERRED, whatever narrowed it. A deferring run built + // some of the relations its manifest names and resolved the rest out of the + // state it read, so publishing that manifest would record relations nothing + // built — and a model renamed since would be recorded under a name only a + // full build creates, breaking every later deferral until one repairs it. + // `publishes_ownership` cannot see this on its own: it reads the caller's + // overrides, and a descriptor that already narrows `select` needs none. + if run.is_ok() + && command == "build" + && inv.deferral.is_none() + // A run of the DEPLOYED version, by kind. A preview carries a + // caller-supplied `script_hash` into `runnable_id` + // (`run_preview_script`), so the version guard alone would let anyone who + // may run a job publish arbitrary content as a deployed script's state. + && job.kind == JobKind::Script + && prepared.graph_refresh.publishes_ownership() + { + // Losing it costs the next deferral, not the run that just finished — + // but silently, so the one actionable case (an artifact too large for + // the database on an instance with no object storage) says so. + if let Err(e) = crate::dbt_state::publish( + &prepared, + &job.workspace_id, + &job.id, + job.runnable_id.map(|h| h.0), + // An attempt was spent, so `run_results.json` on disk is the one + // `dbt retry` left: the nodes it redid, not the build. + node_retry.is_some_and(|p| retries_left < p.attempts()), + conn, + ) + .await + { + append_logs( + &job.id, + &job.workspace_id, + format!("\nCould not publish this run as the environment's dbt state: {e}\n"), + conn, + ) + .await; + } + } let reconciled = reconcile_materializations(&prepared, &results, job, conn, client).await; terminalize_running_relations(job, &reconciled, conn).await; @@ -653,12 +826,26 @@ pub(crate) async fn dbt_dep( None => GraphPublisher::Unversioned, }; let superseded = if let Some(warehouse) = prepared.warehouse.as_deref() { - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, prepared.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + &prepared, + &descriptor, + &inv, + // A deploy resolves the project by parsing it; nothing is built, so + // the pass takes the descriptor's own answer. + "parse", + &mut ctx, + job_id, + w_id, + &conn, + ) + .await?; let published = persist_ingest( db, w_id, @@ -682,6 +869,7 @@ pub(crate) async fn dbt_dep( &conn, ) .await; + warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await; } !published } else { @@ -866,6 +1054,9 @@ impl GraphRefresh { if selection_is_overridden(descriptor, args)? { self.per_run_models = true; } + if full_refresh_is_overridden(descriptor, args)? { + self.per_run_models = true; + } Ok(()) } } @@ -894,6 +1085,17 @@ pub struct PreparedProject { /// The descriptor's `profile.target`, passed as `--target` so it applies to /// a project-owned `profiles.yml` as well as a rendered one. pub target: Option, + /// The target dbt actually runs, which is the above only when the descriptor + /// names one: otherwise it is the workspace warehouse's, or the project's own + /// `profiles.yml` default. Half of an environment's identity, since a + /// `target.name` macro decides where a model is built. + pub effective_target: Option, + /// Whether the profile templates where its relations go — a project-owned + /// `profiles.yml`, a `dbt_profile` resource's block, or `profile.schema`, + /// all of which reach dbt as written. Two renderings then share one + /// `relation_root` and an environment cannot be told apart, so such a + /// project neither publishes state nor defers to any. + pub templated_location: bool, /// The profile target's database. Nodes that override it qualify their /// `dbt://` schema segment so two databases cannot collapse onto one node. pub default_database: Option, @@ -931,7 +1133,7 @@ impl PreparedProject { /// Where this run's relations live: the resolved schema and database. Drift /// here since the deploy means the stored graph names relations that no /// longer exist. - fn relation_root(&self) -> String { + pub(crate) fn relation_root(&self) -> String { format!( "{}|{}", self.default_schema.as_deref().unwrap_or(""), @@ -1063,8 +1265,8 @@ pub(crate) async fn prepare_project( .chain(invocation_env.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect(); - let (profiles_dir, warehouse, adapter, default_database, default_schema, profile_digest) = - write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let profile = write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let adapter = profile.adapter.clone(); // The lockfile's version, when it pinned one for this same engine — a // descriptor edited to another engine invalidates the pin. let pinned_version = locks @@ -1185,18 +1387,20 @@ pub(crate) async fn prepare_project( h.finish() }, sandbox_config, - profile_digest, + profile_digest: profile.digest, project_dir, - profiles_dir, + profiles_dir: profile.dir, engine, graph_refresh, - warehouse, + warehouse: profile.warehouse, target: descriptor.profile.target.clone(), + effective_target: profile.target, + templated_location: profile.templated_location, descriptor_content: descriptor_content.to_string(), descriptor_env, - default_database, - default_schema, + default_database: profile.database, + default_schema: profile.schema, script_path: script_path.to_string(), env, }; @@ -1511,6 +1715,24 @@ async fn strip_git_remote(dir: &Path) -> std::io::Result<()> { tokio::fs::write(&config, out).await } +/// What resolving the run's connection settled, beyond the file itself. +struct ResolvedProfile { + dir: PathBuf, + /// The workspace warehouse's NAME, when this project belongs to one. + warehouse: Option, + adapter: DbtAdapter, + database: Option, + schema: Option, + /// The target dbt actually runs, which is not always the descriptor's: it + /// falls back to the workspace warehouse's, and to the project's own + /// `profiles.yml` default. Resolved because it is half of an environment's + /// identity and a `target.name` macro can move every relation. + target: Option, + /// Whether a project-owned `profiles.yml` templates where its relations go. + templated_location: bool, + digest: String, +} + /// Write `profiles.yml`, either rendered from a Windmill resource or taken from /// the project itself. Both paths are supported (decision 8): the workspace /// warehouse is the ergonomic one, the project's own file is what makes an @@ -1521,14 +1743,7 @@ async fn write_profiles( job_dir: &str, client: &AuthedClient, template_env: &HashMap, -) -> error::Result<( - PathBuf, - Option, - DbtAdapter, - Option, - Option, - String, -)> { +) -> error::Result { // The workspace's warehouse, always: a descriptor names one by NAME or takes // `main`, and cannot name a resource at all. The NAME is what asset identity // keys on, so every project on one warehouse shares its nodes while the @@ -1609,14 +1824,16 @@ async fn write_profiles( } None => None, }; - return Ok(( + return Ok(ResolvedProfile { dir, - identity, + warehouse: identity, adapter, - target.database, - target.schema, - profile_digest, - )); + database: target.database, + schema: target.schema, + target: Some(target.name), + templated_location: target.templated_location, + digest: profile_digest, + }); } use windmill_common::workspaces::DBT_PROFILE_RESOURCE_TYPE; @@ -1711,14 +1928,22 @@ async fn write_profiles( rendered.root_certificate_pem.as_deref(), &client.token, ); - Ok(( + Ok(ResolvedProfile { dir, - Some(warehouse.to_string()), + warehouse: Some(warehouse.to_string()), adapter, - rendered.database, - rendered.schema, - profile_digest, - )) + // A `dbt_profile` resource is one block of the user's own + // `profiles.yml`, copied through unchanged, and `profile.schema` is + // written as given — so either can be a template dbt renders and this + // runtime does not, exactly as a project-owned file can. + templated_location: [rendered.database.as_deref(), rendered.schema.as_deref()] + .iter() + .any(|v| v.is_some_and(is_jinja)), + database: rendered.database, + schema: rendered.schema, + target: Some(target.to_string()), + digest: profile_digest, + }) } /// Where a workspace warehouse name points: its resource path and, if the @@ -1859,13 +2084,45 @@ async fn adapter_from_profiles_yml( // identically to one on a workspace warehouse, which is what lets the two // meet on the same node when they are on the same relation. let (database_key, schema_key) = adapter.target_identity_keys(); - let read = |k: &str| { + let raw = |k: &str| { out.get(k) .and_then(|v| v.as_str()) - .map(|v| v.to_string()) - .filter(|v| !v.is_empty() && !v.contains("{{")) + .filter(|v| !v.is_empty()) }; - Ok(ProfileTarget { adapter, database: read(database_key), schema: read(schema_key) }) + let read = |k: &str| raw(k).filter(|v| !v.contains("{{")).map(|v| v.to_string()); + Ok(ProfileTarget { + adapter, + database: read(database_key), + schema: read(schema_key), + // A TEMPLATED location is one dbt renders and this runtime does not, so + // two renderings of this file resolve to one `relation_root` and would + // share one environment — `{{ }}` because `read` drops it and it reads + // as absent, `{% %}` because the raw block is kept and reads the same + // for every rendering. Distinguished from plainly absent, which is the + // adapter's default and does not move. + templated_location: [database_key, schema_key] + .iter() + .any(|k| raw(k).is_some_and(is_jinja)), + // The output actually chosen, which for a templated `target:` is the sole + // one rather than the template text no output answers to. + name: match ( + templated_target, + outputs.as_mapping().and_then(|m| m.keys().next()), + ) { + (true, Some(only)) => only.as_str().unwrap_or(target).to_string(), + _ => target.to_string(), + }, + }) +} + +/// Whether dbt would RENDER this value rather than take it literally. +/// +/// Both delimiters, because dbt renders a profile through Jinja: `{{ … }}` +/// substitutes and `{% … %}` branches, and a schema spelled +/// `{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}` moves +/// every relation exactly as an `env_var()` does. +fn is_jinja(v: &str) -> bool { + v.contains("{{") || v.contains("{%") } /// What a project-owned `profiles.yml` target says, for the two things Windmill @@ -1876,6 +2133,14 @@ struct ProfileTarget { adapter: DbtAdapter, database: Option, schema: Option, + /// The output this resolved to, by name. + name: String, + /// Whether its database or schema is a template rather than a literal. The + /// fields above cannot say: a `{{ }}` value is dropped and reads as absent, + /// a `{% %}` block is kept and reads the same for every rendering. So this + /// is what separates "the adapter's default, which does not move" from + /// "wherever this run's environment renders it to". + templated_location: bool, } lazy_static::lazy_static! { @@ -2191,6 +2456,29 @@ async fn retry_failed_nodes( } } +/// The flags that point a deferring invocation at its state directory. +/// +/// `--state` is where a deferred `ref()` resolves through — except on a `retry`, +/// which reads the run it RESUMES from that same flag: handed the deferral's +/// directory, dbt resumes the successful run stored there and rebuilds nothing. +/// dbt-core 1.x has `--defer-state` for exactly this split; the Rust engines do +/// not, and a run that defers is refused a retry there rather than rebuilt with +/// its refs resolving into the schema it writes into (`handle_dbt_job`), which +/// is why the last arm never fires in practice. +/// +/// The directory is relative because dbt records the invocation's flags into +/// `run_results.json`: an absolute path would name the job directory of the run +/// being resumed, gone by the time anything reads it back. +fn defer_flags(command: &str, engine: DbtEngine) -> &'static [&'static str] { + match command { + // `--defer` itself is restored with the rest of the resumed + // invocation's arguments and cannot be set from here. + "retry" if engine.has_defer_state_flag() => &["--defer-state", STATE_DIR], + "retry" => &[], + _ => &["--defer", "--state", STATE_DIR], + } +} + #[allow(clippy::too_many_arguments)] async fn run_dbt( p: &PreparedProject, @@ -2212,6 +2500,10 @@ async fn run_dbt( .args(["--log-format-file", "json"]) .args(["--log-level-file", p.engine.engine.progress_log_level()]); + if inv.deferral.is_some() { + cmd.args(defer_flags(command, p.engine.engine)); + } + if with_selection && command != "retry" { add_selection(&mut cmd, descriptor, inv)?; } @@ -2232,8 +2524,7 @@ async fn run_dbt( if let Some(t) = descriptor.threads { cmd.args(["--threads", &t.to_string()]); } - let full_refresh = arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh); - if full_refresh && command != "test" { + if full_refresh(descriptor, inv, command)? { cmd.arg("--full-refresh"); } } @@ -2863,6 +3154,9 @@ async fn run_show( ))); } let mut cmd = dbt_command(p, &["show"]); + if inv.deferral.is_some() { + cmd.args(defer_flags("show", p.engine.engine)); + } add_vars(&mut cmd, descriptor, inv)?; // Intersected with `resource_type:model`, because `show` is only read-only // for models: dbt dispatches a selected SEED through its seed runner and @@ -2882,7 +3176,8 @@ async fn run_show( conn, SHOW_MAX_OUTPUT_BYTES, ) - .await?; + .await? + .stdout; // dbt frames the rows as `{"node": …, "show": [ … ]}`, pretty-printed, with a // banner before and a deprecation summary after — so neither "the line starting // with `{`" nor "first `{` to the end" parses. A streaming deserializer stops at @@ -2956,6 +3251,7 @@ fn build_result( totals, nodes, invocation_args: inv.raw_args.clone(), + deferred_to: inv.deferral.as_ref().map(|d| d.published_by), } } @@ -3064,7 +3360,7 @@ async fn run_parse_only( // manifest and the selection while the warehouse only keys them — so a project // with no warehouse identity still reports what dbt found. The placeholder // reaches no row: the guard below returns before anything is written. - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, p.warehouse.as_deref().unwrap_or("unkeyed"), p.default_database.as_deref(), @@ -3084,6 +3380,20 @@ async fn run_parse_only( else { return Ok(to_raw_value(&result)); }; + // AFTER the guard: the pass is a second `dbt compile` and a parquet decode, + // and a parse that stores nothing has nowhere to put what it would produce. + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + "parse", + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; match conn { Connection::Sql(db) => match job.runnable_id.map(|h| h.0) { Some(script_hash) => { @@ -3143,11 +3453,70 @@ async fn run_parse_only( Ok(to_raw_value(&result)) } +/// Fold this project's column lineage into the graph about to be stored, when +/// the descriptor asked for it. +/// +/// One helper for all three ingests — deploy, editor parse, per-run refresh — +/// because a graph that carries column lineage in one provenance and not another +/// reads as the lineage having disappeared. +async fn attach_column_index( + ingested: &mut windmill_common::dbt_manifest::IngestedManifest, + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result<()> { + // The nodes this graph kept, so the pass reads only rows it could store: the + // index describes the whole project, this graph one selection of it. + let kept: std::collections::HashSet<&str> = ingested + .nodes + .iter() + .map(|n| n.unique_id.as_str()) + .collect(); + let index = + crate::dbt_column_index::collect( + p, descriptor, inv, command, ctx, job_id, w_id, conn, &kept, + ) + .await?; + drop(kept); + let Some(index) = index else { + return Ok(()); + }; + let found = index.edges.len(); + ingested.attach_column_index(index); + let kept = ingested.column_edges.len(); + let typed: usize = ingested + .nodes + .iter() + .filter(|n| n.column_schema.is_some()) + .count(); + // Counted here rather than at the pass: the index describes the whole + // project and this graph describes one selection of it, so `found` is what + // dbt produced and `kept` is what the graph can draw. + let dropped = match found.saturating_sub(kept) { + 0 => String::new(), + n => format!(" ({n} outside this graph or past the cap)"), + }; + append_logs( + job_id, + w_id, + format!("\nIngested {kept} column lineage edges{dropped} and typed {typed} nodes\n"), + conn, + ) + .await; + Ok(()) +} + /// Refresh the stored graph from the manifest this run produced. async fn ingest_from_run( p: &PreparedProject, descriptor: &DbtDescriptor, inv: &Invocation, + command: &str, ctx: &mut JobCtx<'_>, job: &MiniPulledJob, conn: &Connection, @@ -3164,12 +3533,24 @@ async fn ingest_from_run( // filter this run's manifest by a different node set than it built. let selected = resolve_selection(p, descriptor, inv, ctx, &job.id, &job.workspace_id, conn).await?; - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, p.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + command, + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; // Only a run whose models are its own snapshots per run. A static // descriptor at a moved profile re-ingests the VERSION's graph, since the // move outlives the run; one that neither drifted nor overrode anything @@ -3177,7 +3558,7 @@ async fn ingest_from_run( let snapshot_job = p.graph_refresh.snapshot_job(job.id); match conn { Connection::Sql(db) => { - persist_ingest( + let published = persist_ingest( db, &job.workspace_id, script_path, @@ -3190,6 +3571,13 @@ async fn ingest_from_run( p.graph_refresh.publishes_ownership(), ) .await?; + // Publishing ownership from a RUN makes this project the owner of + // those relations exactly as a deploy does, so it can be what leaves + // a subscription accepted while the relation had no producer with dbt + // as its only one. Same warning the deploy emits. + if published && p.graph_refresh.publishes_ownership() { + warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await; + } } // An agent worker reaches these tables only through the API. Publishing // is the whole of what it needs: a worker that can replace the graph @@ -3242,7 +3630,7 @@ enum GraphPublisher { /// Replace this script's graph, unless a newer version of it has been deployed. /// /// Write one ingest: the sidecar rows and the `asset` usages the manifest -/// implies. No subscriptions — a `dbt://` one could never fire. +/// implies. No subscriptions — a dbt project is not woken by the cascade. /// /// Returns whether this job was still the one entitled to the path-keyed half — /// false once a newer version has superseded it, or once the version is gone. @@ -3333,9 +3721,10 @@ async fn persist_ingest( &ingested.assets, ) .await?; - // A `dbt://` subscription can never fire, so none are derived from the - // manifest. The delete stays to clear what earlier versions wrote, which would - // otherwise keep drawing cascade arrows that wake nothing. + // A dbt project is not woken by the asset cascade (refused at deploy), so + // none are derived from the manifest either. The delete stays to clear what + // earlier versions wrote, which would otherwise keep drawing cascade arrows + // that wake nothing. sqlx::query!( "DELETE FROM script_trigger WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2 @@ -3349,6 +3738,56 @@ async fn persist_ingest( Ok(true) } +/// Log the `// on dbt://…` subscriptions this project's relations leave dormant. +/// +/// Subscribing to a relation dbt already owns is refused at the subscriber's +/// deploy, but one deployed while nothing produced that relation is accepted — +/// as it is for every other asset kind — and an ingest is what can afterwards +/// make dbt its only producer. A dbt run does not dispatch, so such an edge is +/// drawn on the canvas and never fires; a job's own log is where that ordering is +/// visible. +/// +/// Called from both points that publish ownership, the deploy and a run whose +/// static descriptor found its profile moved — either can be the one that claims +/// the relation. +async fn warn_dormant_subscribers( + db: &sqlx::Pool, + w_id: &str, + job_id: &Uuid, + ingested: &windmill_common::dbt_manifest::IngestedManifest, + conn: &Connection, +) { + use windmill_common::assets::AssetUsageAccessType; + let relations: Vec = ingested + .assets + .iter() + .filter(|a| { + matches!( + a.access_type.or(a.alt_access_type), + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) + }) + .map(|a| a.path.clone()) + .collect(); + match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await { + Ok(edges) if !edges.is_empty() => { + append_logs( + job_id, + w_id, + format!( + "\nThese subscriptions will not fire — a dbt run does not trigger downstream \ + runs, and nothing else writes their relation:\n {}\n", + edges.join("\n ") + ), + conn, + ) + .await; + } + Ok(_) => {} + Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"), + } +} + /// Serialize publishers for one script path and confirm this job's version is /// still the newest. Both happen inside the caller's transaction, so a newer /// publisher either commits before this check sees it, or waits behind it and @@ -3411,6 +3850,12 @@ async fn resolve_selection( return Ok(None); } let mut cmd = dbt_command(p, &["ls"]); + // The same state the build resolves through, or a `result:` selector — which + // reads `run_results.json` out of it, and which `select` passes to dbt + // verbatim — fails here, before the build that would have honoured it. + if inv.deferral.is_some() { + cmd.args(defer_flags("ls", p.engine.engine)); + } // A project whose models call `var()` without a default fails to parse // without these, so the selection resolver needs them exactly as the run // does. Placeholders that only a run can fill are dropped rather than @@ -3424,11 +3869,15 @@ async fn resolve_selection( } cmd.args(["--output", "json", "--quiet"]); add_selection(&mut cmd, descriptor, inv)?; + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; // Captured directly, not through `handle_child`: its `pipe_stdout` path goes // through the job-log writer, which `NO_LOGS_AT_ALL` discards — the selection // would resolve to the empty set and the ingest would wipe the script's assets // while dbt went on building the descriptor's models. - let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES).await?; + let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES) + .await? + .stdout; let mut set = std::collections::HashSet::new(); for line in stdout.lines() { let line = line.trim(); @@ -3441,15 +3890,37 @@ async fn resolve_selection( } } } - if set.is_empty() { - // A selection that matches nothing would be ingested as "this script - // owns no relations", wiping its graph and cascade edges — the same - // outcome a failed capture produces, and indistinguishable from it. - // Refuse rather than silently un-wire the script. + // Empty is a real answer from a `state:` or `result:` method and from nothing + // else: `state:modified+` matches nothing exactly when nothing changed since + // the published state, and a run with no work to do is a successful one. Any + // other selection matching nothing is a selector that names nothing — a + // misspelled model, say — which must not pass as a build that did its job. + // Exempting by ORIGIN rather than by method would let every such typo through. + // + // What makes the exemption safe is that the empty set is never ingested as + // ownership, and that now holds through `check_state_selectors`: a `state:` + // or `result:` method survives it only from a run's OWN selection, which + // makes `add_caller_args` set `per_run_models`, which makes + // `publishes_ownership()` false, so the run stores a snapshot of its own. + // Relax the descriptor arm there and a descriptor-narrowed `state:modified+` + // reaches here on an unchanged project and wipes the graph the `else` below + // guards, with nothing failing. + if set.is_empty() && !selection_names(&select, &exclude, &["state", "result"]) { return Err(Error::ExecutionErr( - "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection rather \ - than deploying a script that owns nothing" - .to_string(), + if selection_is_overridden(descriptor, &inv.args)? { + "this run's `select`/`exclude` matched no dbt nodes, so it would build nothing; \ + check the selector. Only a `state:` or `result:` selector may match nothing, \ + its empty answer being a real one" + .to_string() + } else { + // The descriptor's is also ingested as "this script owns no + // relations", wiping its graph and cascade edges — the same + // outcome a failed capture produces, and indistinguishable from + // it. Refuse rather than silently un-wire the script. + "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection \ + rather than deploying a script that owns nothing" + .to_string() + }, )); } Ok(Some(set)) @@ -3459,6 +3930,34 @@ async fn resolve_selection( /// what is kept is the TAIL, because dbt prints its error summary last. const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; +/// What a captured invocation produced. `stderr` is where dbt writes its +/// diagnostics — the errors and warnings block — so a caller that has to explain +/// a SUCCESSFUL run needs it as much as a failing one does. +pub(crate) struct Captured { + pub stdout: String, + pub stderr: String, + /// Whether the child exited zero. Separate from the `Result` on purpose: an + /// `Err` from `run_captured` is the JOB's — a cancellation or its deadline — + /// so a caller that tolerates a failed command must still propagate one. + pub success: bool, + /// Whether the output ceiling cut the child short. Only ever true under + /// [`Overflow::Truncate`]. + pub truncated: bool, +} + +/// What an over-long stdout means to the caller. +/// +/// The ceiling belongs to the PASS, not to the job: a caller that only annotates +/// a job wants to keep what it read and carry on, while one whose whole result +/// is that output has nothing to return without it. +#[derive(PartialEq, Eq, Clone, Copy)] +pub(crate) enum Overflow { + /// Fail the job. For a command whose output IS the answer. + Fail, + /// Stop reading, kill the child, and report `truncated`. + Truncate, +} + /// Run a command for its stdout under the job's cancellation and timeout. /// The same poller `handle_child` uses drives them, so a cancel or a deadline /// drops the wait future — which owns the child, and `kill_on_drop` then @@ -3471,7 +3970,7 @@ const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; /// never holds more than it, so it has to be enforced while reading. Both pipes /// are drained concurrently because a child that fills the one nobody reads /// blocks forever. -async fn run_capturing( +pub(crate) async fn run_captured( mut cmd: Command, name: &str, ctx: &mut JobCtx<'_>, @@ -3479,7 +3978,8 @@ async fn run_capturing( w_id: &str, conn: &Connection, max_stdout_bytes: usize, -) -> error::Result { + on_overflow: Overflow, +) -> error::Result { use tokio::io::AsyncReadExt; let mut child = cmd @@ -3514,6 +4014,7 @@ async fn run_capturing( let mut out_buf = vec![0u8; 16 * 1024]; let mut err_buf = vec![0u8; 16 * 1024]; let (mut out_open, mut err_open) = (true, true); + let mut truncated = false; while out_open || err_open { tokio::select! { r = stdout_pipe.read(&mut out_buf[..]), if out_open => match r { @@ -3521,14 +4022,19 @@ async fn run_capturing( Ok(n) => { if stdout.len() + n > max_stdout_bytes { // Killed here rather than left to `kill_on_drop` - // so the child is gone before the error unwinds, - // not merely once this future is dropped. + // so the child is gone before this returns, not + // merely once the future is dropped. let _ = child.kill().await; - return Err(Error::ExecutionErr(format!( - "{name} produced more than {} MB of output. Narrow the \ - selection, or query the relation from a SQL script.", - max_stdout_bytes / 1024 / 1024 - ))); + if on_overflow == Overflow::Fail { + return Err(Error::ExecutionErr(format!( + "{name} produced more than {} MB of output. Narrow the \ + selection, or query the relation from a SQL script.", + max_stdout_bytes / 1024 / 1024 + ))); + } + truncated = true; + out_open = false; + continue; } stdout.extend_from_slice(&out_buf[..n]); } @@ -3551,7 +4057,7 @@ async fn run_capturing( .wait() .await .map_err(|e| Error::internal_err(format!("{name} failed: {e}")))?; - Ok((status, stdout, stderr)) + Ok((status, stdout, stderr, truncated)) }, ctx.worker_name, w_id, @@ -3561,14 +4067,46 @@ async fn run_capturing( })), ) .await?; - let (status, stdout, stderr) = out; - if !status.success() { + let (status, stdout, stderr, truncated) = out; + Ok(Captured { + stdout: String::from_utf8_lossy(&stdout).to_string(), + stderr: String::from_utf8_lossy(&stderr).to_string(), + // A killed child reports failure; under `Truncate` that is the ceiling's + // doing, not the project's, and the caller reads `truncated` to tell. + success: status.success(), + truncated, + }) +} + +/// `run_captured`, with a non-zero exit folded into the error — what a caller +/// that needs the command to have WORKED wants. +pub(crate) async fn run_capturing( + cmd: Command, + name: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + max_stdout_bytes: usize, +) -> error::Result { + let captured = run_captured( + cmd, + name, + ctx, + job_id, + w_id, + conn, + max_stdout_bytes, + Overflow::Fail, + ) + .await?; + if !captured.success { return Err(Error::ExecutionErr(format!( "{name} failed: {}", - String::from_utf8_lossy(&stderr) + captured.stderr ))); } - Ok(String::from_utf8_lossy(&stdout).to_string()) + Ok(captured) } /// Run a preparation command through the same child handler the build uses, so @@ -3761,7 +4299,7 @@ async fn save_run_state( if let Connection::Sql(db) = conn { { // Only while a live dbt version stays at this path — the test - // `clear_dbt_run_state_if_path_retired` retires state by, plus the + // `clear_dbt_script_state_if_path_retired` retires state by, plus the // language, since a rename leaves the old path archived rather than // deleted and a path can come back as another language. A job already // running finishes after those move or clear the row: writing then @@ -3923,6 +4461,12 @@ pub struct Invocation { /// what it pointed at must not. pub raw_args: HashMap>, pub envs: HashMap, + /// The stored dbt state this invocation resolves an unbuilt `ref()` through, + /// materialised into the job directory. Carried here rather than passed to + /// each phase: the model phase, the `after_all` tests and every in-job node + /// retry must all resolve a `ref()` the same way, or the tests assert against + /// relations the models never read. + pub deferral: Option, /// A run must fail on a `{{ }}` placeholder it cannot fill; a deploy, which /// has no arguments at all, tolerates them. Declared rather than inferred /// from the argument count: a run submitted with `{}` is still a run, and @@ -4075,11 +4619,12 @@ async fn restore_from_db( if !has_retryable_node(&row.run_results) { return Err(nothing_to_retry()); } - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - tokio::fs::write(target.join("run_results.json"), &row.run_results) - .await - .map_err(|e| Error::internal_err(format!("restoring run_results.json: {e}")))?; + write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&row.run_results), + StateManifest::None, + ) + .await?; // No manifest came with the row, so one has to be re-derived — but not here: // these arguments are as SUBMITTED, and a `$var:` in them shapes the graph // only once resolved. The caller resolves, then parses. @@ -4311,20 +4856,16 @@ async fn restore_run_state( return Err(different_project()); } let saved_args_digest = saved_args_digest.map(str::to_string); - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - // From the bytes already read, not by copying the file again: a burst of saves - // can prune this generation mid-restore, and a `dbt retry` whose + // The results go from the bytes already read, not by copying the file again: a + // burst of saves can prune this generation mid-restore, and a `dbt retry` whose // `run_results.json` went missing rebuilds nothing and reports success. The // manifest has no such copy, so a failure there falls back to a `dbt parse`. - tokio::fs::write(target.join("run_results.json"), &saved_results) - .await - .map_err(|e| { - Error::internal_err(format!("could not restore the previous run's results: {e}")) - })?; - let needs_parse = tokio::fs::copy(snapshot.join("manifest.json"), target.join("manifest.json")) - .await - .is_err(); + let needs_parse = !write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&saved_results), + StateManifest::CopyOf(snapshot.join("manifest.json")), + ) + .await?; // The generation was chosen from a row read before the file work above. A run // finishing in that window publishes a newer one, and resuming the superseded // generation redoes nodes it has already rebuilt — appending to an incremental @@ -4518,7 +5059,11 @@ fn has_retryable_node(run_results: &str) -> bool { } /// Append `--vars` if the descriptor (or the run) declares any. -fn add_vars(cmd: &mut Command, descriptor: &DbtDescriptor, inv: &Invocation) -> error::Result<()> { +pub(crate) fn add_vars( + cmd: &mut Command, + descriptor: &DbtDescriptor, + inv: &Invocation, +) -> error::Result<()> { let vars = resolved_vars(descriptor, &inv.args, inv.strict)?; if !vars.is_empty() { cmd.args(["--vars", &serde_json::to_string(&vars).unwrap_or_default()]); @@ -4713,10 +5258,26 @@ fn add_selection( descriptor: &DbtDescriptor, inv: &Invocation, ) -> error::Result<()> { - for s in effective_select(descriptor, inv)? { + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; + // The seam itself, which the DEPLOY reaches without going through a run: it + // resolves the descriptor's selection to decide what the script owns, and + // never computes a `defer`. A run has been checked earlier, where the message + // can still come before the state fetch. + check_state_selectors( + &select, + &exclude, + if inv.deferral.is_some() { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(descriptor, &inv.args)?, + )?; + for s in select { cmd.args(["--select", &s]); } - for s in effective_exclude(descriptor, inv)? { + for s in exclude { cmd.args(["--exclude", &s]); } if let Some(sel) = effective_selector(descriptor, inv)? { @@ -4725,6 +5286,115 @@ fn add_selection( Ok(()) } +/// The method a selection token names, with the graph operators that can +/// surround a node stripped (`@model`, `+model`, `2+model`, `model+`). +fn selector_method(token: &str) -> Option<&str> { + token + .trim_start_matches('@') + .trim_start_matches(|c: char| c.is_ascii_digit()) + .trim_start_matches('+') + .split_once(':') + .map(|(method, _)| method) +} + +/// Every method a selection names. Each entry is a union of whitespace-separated +/// tokens, and each of those an intersection of comma-separated ones. +fn selection_methods<'a>(entries: &'a [String]) -> impl Iterator { + entries + .iter() + .flat_map(|entry| entry.split([' ', '\t', ','])) + .filter_map(selector_method) +} + +/// Whether the run being checked has the state directory a `state:` or `result:` +/// method reads, or could be given one. +#[derive(Clone, Copy)] +enum StateAccess<'a> { + /// Deferring, so the directory is there. + Given, + /// Not deferring, and `defer` is what would hand it one. + OnRequest, + /// This command resolves a selection without ever deferring, so no setting + /// gives it a state and "turn `defer` on" would be advice that leads nowhere. + Never(&'a str), +} + +/// Whether a selection names any of these methods. +fn selection_names(select: &[String], exclude: &[String], methods: &[&str]) -> bool { + selection_methods(select) + .chain(selection_methods(exclude)) + .any(|method| methods.contains(&method)) +} + +/// Refuse a selection dbt cannot resolve, before it silently resolves to the +/// wrong thing. +/// +/// `state:` and `result:` compare against the artifacts in `--state`, which only +/// a deferring run is given. The engines do not agree on what happens without +/// one: dbt-core 1.x raises, but dbt-sa-cli and fusion read a missing state as an +/// EMPTY one and exit 0, so `state:modified` builds nothing and `state:new` +/// builds the whole project, each as a run that reports success. +/// +/// From the DESCRIPTOR they are refused whether or not the run defers, because +/// that selection also decides which nodes the script owns, and "whatever changed +/// last" is not an ownership answer — the deploy resolves it with no state at all. +/// They describe one run, so they belong in a run's own `select`. +/// +/// `source_status:` compares `sources.json`, which `dbt source freshness` writes +/// and no run publishes here, so it has nothing to compare against under any +/// setting. +/// +/// Only what `select` and `exclude` spell directly: a method reached through a +/// `selectors.yml` definition is named nowhere the worker can read, and dbt's +/// own behaviour is what stands there. +fn check_state_selectors( + select: &[String], + exclude: &[String], + access: StateAccess<'_>, + from_descriptor: bool, +) -> error::Result<()> { + for method in selection_methods(select).chain(selection_methods(exclude)) { + match method { + "source_status" => { + return Err(Error::BadRequest( + "a `source_status:` selector compares the source freshness recorded in \ + `sources.json`, which `dbt source freshness` writes and no run stores \ + here, so there is nothing for it to compare against. Drop the selector" + .to_string(), + )) + } + "state" | "result" if from_descriptor => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector describes what ONE run builds, but the descriptor's \ + selection also decides which nodes this script owns, which a deploy \ + resolves with no state to compare against. Move it to the `select` of a \ + run with `defer` on" + ))) + } + "state" | "result" => match access { + StateAccess::Given => {} + StateAccess::OnRequest => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and only a run with `defer` on is given \ + that state. Turn `defer` on, or drop the selector" + ))) + } + StateAccess::Never(command) => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and `{command}` resolves its selection \ + without building and never defers, so no setting hands it that state. \ + Drop the selector" + ))) + } + }, + _ => {} + } + } + Ok(()) +} + /// Whether this invocation chose its own `select`/`exclude`. /// /// DIFFERENT from the descriptor's, not merely present: `parse_dbt_sig` gives @@ -4748,6 +5418,41 @@ fn selection_is_overridden( Ok(differs("select", &descriptor.select)? || differs("exclude", &descriptor.exclude)?) } +/// Whether this invocation rebuilds incremental models from scratch: the run +/// form's answer when it gave one, else the descriptor's — and never for a +/// `test`, which builds nothing whatever the form said. +/// +/// Shared with the column-lineage pass rather than recomputed there, because +/// `is_incremental()` branches on it: the same model compiles to different SQL — +/// a `{{ this }}` self-join, and any `ref()` inside the incremental branch — so a +/// pass that guessed would describe a build that never ran. +/// +/// `test` returns false because `dbt test` rejects `--full-refresh` outright. +/// It never arrives as a caller's `dbt_command` — the allowlist has no such +/// value — so reading only that allowlist suggests this branch is dead. It is +/// not: `run_dbt` is invoked with `"test"` directly for the `after_all` test +/// phase, and an `after_all` project with `full_refresh: true` reaches here. +pub(crate) fn full_refresh( + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, +) -> error::Result { + if command == "test" { + return Ok(false); + } + Ok(arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh)) +} + +/// Whether this run answered `full_refresh` differently from the deployed +/// descriptor. Like a selection override it changes what the graph describes, +/// since an incremental branch can carry its own `ref()`. +fn full_refresh_is_overridden( + descriptor: &DbtDescriptor, + args: &HashMap>, +) -> error::Result { + Ok(arg_bool(args, "full_refresh")?.is_some_and(|v| v != descriptor.full_refresh)) +} + /// The descriptor's named selector, unless this run named its own selection. /// /// dbt resolves `--selector` INSTEAD of `--select`, so passing both makes the @@ -5738,6 +6443,79 @@ mod tests { .unwrap(); assert!(untouched.publishes_ownership()); assert_eq!(untouched.snapshot_job(job), None); + + // `resolve_selection` lets a selection match nothing on exactly this + // predicate, because a run that scoped its own selection stores a + // snapshot instead of publishing ownership. Should the two ever drift + // apart, an empty caller selection would wipe the script's graph and + // cascade edges, which is the outcome that guard exists to prevent. + // One-directional: a `vars` override also withholds ownership without + // touching the selection, which is why this is an implication and not an + // equivalence. + for args in [ + arg("select", r#"["state:modified+"]"#), + arg("exclude", r#"["tag:nightly"]"#), + ] { + assert!(selection_is_overridden(&descriptor, &args).unwrap()); + let mut g = GraphRefresh::default(); + g.add_caller_args(&descriptor, &args).unwrap(); + assert!( + !g.publishes_ownership(), + "an overridden selection must not publish ownership" + ); + } + + // `full_refresh` decides whether `is_incremental()` is true, so an + // incremental model's self-join — and any `ref()` inside that branch — + // exists in one answer and not the other. A run that flips it describes + // a different graph, and gets its own. + let mut refreshed = GraphRefresh::default(); + refreshed + .add_caller_args(&descriptor, &arg("full_refresh", "true")) + .unwrap(); + assert!(refreshed.needed()); + assert_eq!(refreshed.snapshot_job(job), Some(job)); + + // The same echo rule: the form posts the descriptor's own value back on + // every run, and reading that as an override would make each one + // caller-scoped. + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let mut echoed_flag = GraphRefresh { profile_drift: true, ..Default::default() }; + echoed_flag + .add_caller_args(&always, &arg("full_refresh", "true")) + .unwrap(); + assert_eq!(echoed_flag.snapshot_job(job), None); + } + + /// The build and the analysis pass read this through one function, so they + /// cannot disagree about which SQL the run compiles — including for `test`, + /// which rebuilds nothing whatever the descriptor or the form said. + #[test] + fn full_refresh_is_one_answer_for_the_build_and_the_pass() { + let inv = |args: HashMap>| Invocation { + args, + raw_args: Default::default(), + envs: Default::default(), + strict: true, + deferral: None, + }; + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let never = DbtDescriptor::default(); + let on = HashMap::from([( + "full_refresh".to_string(), + RawValue::from_string("true".to_string()).unwrap(), + )]); + + assert!(full_refresh(&always, &inv(Default::default()), "build").unwrap()); + assert!(!full_refresh(&never, &inv(Default::default()), "build").unwrap()); + assert!( + full_refresh(&never, &inv(on), "build").unwrap(), + "the form's answer wins over the descriptor's" + ); + assert!( + !full_refresh(&always, &inv(Default::default()), "test").unwrap(), + "a test builds nothing, so neither the build nor the pass may pass the flag" + ); } // `dbt retry` restores the previous run's target/ from this directory, so two @@ -5766,6 +6544,128 @@ mod tests { ); } + // A profile whose location dbt renders cannot be told apart from another + // rendering of itself, so it neither publishes state nor defers. Both + // delimiters count: a conditional block moves a schema exactly as an + // `env_var()` substitution does. + #[test] + fn a_rendered_profile_location_is_recognised_by_either_delimiter() { + assert!(is_jinja("{{ env_var('DBT_SCHEMA') }}")); + assert!(is_jinja( + "{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}" + )); + assert!(!is_jinja("analytics")); + assert!(!is_jinja("")); + } + + // dbt-sa-cli and fusion exit 0 on a state selector with no state, so nothing + // downstream would report this: the graph operators have to be stripped for + // the method to be seen at all. + #[test] + fn a_state_selector_is_found_under_any_graph_operator() { + // A run's own selection, which is the only place these belong. + let refused = |sel: &str, access: StateAccess<'_>| { + check_state_selectors(&[sel.to_string()], &[], access, false).is_err() + }; + for sel in [ + "state:modified", + "state:modified+", + "+state:new", + "@state:modified", + "2+state:modified+3", + "tag:nightly,state:modified", + "stg_orders+ result:error+", + ] { + assert!( + refused(sel, StateAccess::OnRequest), + "{sel} should need `defer`" + ); + assert!( + !refused(sel, StateAccess::Given), + "{sel} should pass while deferring" + ); + // A parse resolves a selection without ever deferring, so it is + // refused where a run would have been told to turn `defer` on. + assert!( + refused(sel, StateAccess::Never("parse")), + "{sel} cannot parse" + ); + // The descriptor's selection also decides what the script owns, and + // the deploy resolves it with no state, so deferring cannot save it. + assert!( + check_state_selectors(&[sel.to_string()], &[], StateAccess::Given, true).is_err(), + "{sel} should never be a descriptor selection" + ); + } + // A node whose name merely starts with a method's letters is not one. + for sel in ["stg_orders+", "tag:nightly", "stateful_model+"] { + assert!( + !refused(sel, StateAccess::OnRequest), + "{sel} is not a state selector" + ); + } + // No run publishes `sources.json`, so deferring does not help. + for access in [ + StateAccess::Given, + StateAccess::OnRequest, + StateAccess::Never("parse"), + ] { + assert!(refused("source_status:fresher+", access)); + } + // `exclude` reaches dbt the same way `select` does. + assert!(check_state_selectors( + &[], + &["state:modified".to_string()], + StateAccess::OnRequest, + false + ) + .is_err()); + + // The same recognition decides which empty selections `resolve_selection` + // lets through. Only these two answer "nothing" meaningfully; a selector + // naming nothing must not pass as a build that did its work. + const STATE_BACKED: &[&str] = &["state", "result"]; + for sel in ["state:modified+", "result:error+", "tag:x,state:new"] { + assert!( + selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + for sel in [ + "mispelled_model", + "tag:nightly", + "stg_orders+", + "source_status:fresher+", + ] { + assert!( + !selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + } + + // The one flag choice that is silently wrong rather than loudly wrong: a + // `retry` handed `--state` resumes the SUCCESSFUL run stored there and + // rebuilds nothing, reporting a green retry of a failed run. + #[test] + fn a_retry_is_never_handed_the_deferral_as_its_state() { + assert_eq!( + defer_flags("build", DbtEngine::DbtCore1x), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("test", DbtEngine::Fusion), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("retry", DbtEngine::DbtCore1x), + ["--defer-state", crate::dbt_state::STATE_DIR] + ); + for engine in [DbtEngine::DbtCore2x, DbtEngine::Fusion] { + assert!(defer_flags("retry", engine).is_empty()); + } + } + #[test] fn events_without_a_relation_are_not_materializations() { // A test node has no relation of its own. diff --git a/backend/windmill-worker/src/dbt_state.rs b/backend/windmill-worker/src/dbt_state.rs new file mode 100644 index 0000000000..ecf3d0dbe3 --- /dev/null +++ b/backend/windmill-worker/src/dbt_state.rs @@ -0,0 +1,748 @@ +//! The dbt state a project last built into one environment, and the state +//! directory a run reads it back through. +//! +//! `dbt --defer --state ` resolves a `ref()` the run does not build to the +//! relation the manifest in `` names, instead of to the schema this run +//! writes into. That makes the state a durable, per-environment artifact rather +//! than a cache: the next run of a project usually lands on a worker holding +//! neither the manifest nor the results, so anything worker-local answers for +//! one machine's history rather than for the environment. +//! +//! Two artifacts live in that directory and both are stored: `manifest.json`, +//! which is what a deferral resolves through, and `run_results.json`, which +//! `select`'s `result:` selectors read — and `select` reaches dbt verbatim, so a +//! state directory missing it fails a selection a user may legitimately write. + +use std::path::{Path, PathBuf}; + +use uuid::Uuid; +use windmill_common::error::{self, Error}; +use windmill_common::worker::Connection; + +use crate::dbt_executor::{digest, PreparedProject, ARTIFACTS_DIR}; + +lazy_static::lazy_static! { + /// Above this, an artifact goes to the instance's object storage instead of + /// into the row. A manifest passes a few hundred KB on a handful of models + /// and grows with the project, so this ceiling is what decides whether a + /// large project needs storage configured at all; a small one stays in the + /// database, where it costs no round trip and needs nothing configured. + static ref DBT_STATE_INLINE_MAX_BYTES: usize = std::env::var("DBT_STATE_INLINE_MAX_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8 * 1024 * 1024); +} + +/// The directory `--state` points at. Inside the job directory, so it sits in +/// the sandbox's one writable bind and goes away with the job, and prefixed like +/// the artifacts directory beside it so a project carrying a directory of this +/// name is not overwritten. +/// +/// Passed to dbt RELATIVE, and that is load-bearing rather than tidiness. dbt +/// records the invocation's flags into `run_results.json` and a later +/// `dbt retry` restores them, so an absolute path would name the job directory +/// of the run being resumed — gone by then, leaving the retry to resolve a +/// deferred `ref()` against nothing. Relative, it resolves against the project +/// root, which is whichever job directory the retry landed in. +pub(crate) const STATE_DIR: &str = "wm_dbt_state"; + +/// Where this run's relations live, which is the only thing a deferral is about. +pub(crate) fn environment(p: &PreparedProject) -> String { + environment_key( + p.warehouse.as_deref(), + // The target dbt RUNS, not the descriptor's: it falls back to the + // workspace warehouse's and to the project's own default, so reading the + // descriptor's would put two inherited targets under one empty name. + p.effective_target.as_deref(), + // The pair `relation_root` reports to the graph's drift check, taken + // apart so neither can absorb the other's delimiter below. + p.default_schema.as_deref(), + p.default_database.as_deref(), + ) +} + +/// The warehouse and the target name the environment; the database and schema +/// they resolve to are in the key because a repointed warehouse resource or a +/// moved schema keeps both names while putting the relations somewhere else — +/// and a manifest is a list of relation names, so a deferral has no other way to +/// notice. A move therefore reads as an environment nothing has published yet. +/// +/// Length-prefixed rather than joined on a separator. Every component but the +/// warehouse is spelled by the user — a dbt target name and a schema are both +/// arbitrary strings a profile may quote — so `prod|analytics` + `scratch` and +/// `prod` + `analytics|scratch` would otherwise be one key, and a profile moving +/// between them would read as the same environment rather than as one nothing +/// has published. Same reasoning as `stable_digest`, and still legible in a row: +/// `4:main|4:prod|9:analytics|12:dbt_wh_defer`. What a MESSAGE names is +/// `environment_label`, since this encoding is for storage. +fn environment_key( + warehouse: Option<&str>, + target: Option<&str>, + schema: Option<&str>, + database: Option<&str>, +) -> String { + [warehouse, target, schema, database] + .iter() + .map(|v| { + let v = v.unwrap_or(""); + format!("{}:{v}", v.len()) + }) + .collect::>() + .join("|") +} + +/// The environment as a message names it: the key above is length-prefixed for +/// storage, which is not something to put in front of a caller. +pub(crate) fn environment_label(p: &PreparedProject) -> String { + format!( + "warehouse `{}`, target `{}`, relations in `{}`", + p.warehouse.as_deref().unwrap_or("(none)"), + p.effective_target + .as_deref() + .unwrap_or("(the profile's default)"), + match (p.default_database.as_deref(), p.default_schema.as_deref()) { + (Some(db), Some(schema)) => format!("{db}.{schema}"), + (None, Some(schema)) => schema.to_string(), + _ => "(the adapter's default)".to_string(), + } + ) +} + +/// The state one environment last published. +pub(crate) struct StoredState { + pub manifest: String, + pub run_results: Option, + /// The run that published it, so a deferring run can say what it deferred to. + pub job_id: Uuid, +} + +/// Publish this run's artifacts as the environment's state. +/// +/// Called for a run that BUILT what the script's own descriptor selects and +/// succeeded (see `handle_dbt_job`). Best-effort in the same sense as the retry +/// state: losing it costs the next deferral, not the run that just finished. +/// +/// **What the artifacts may carry follows from that condition.** A publishing run +/// added nothing of its own — no `select` or `vars` override, and a descriptor +/// interpolating a `{{ }}` placeholder into `vars` never publishes at all — so +/// dbt's `run_results.json` records the descriptor's own arguments, which are the +/// script's content. That is why this is keyed by environment where +/// `dbt_run_state` is keyed by principal: the retry state holds whatever a caller +/// submitted, this holds what the script says. Widen the publish condition and +/// that stops being true. +pub(crate) async fn publish( + p: &PreparedProject, + w_id: &str, + job_id: &Uuid, + // The version this job ran. `None` for a preview, which publishes nothing. + script_hash: Option, + // A build recovered by the automatic in-job node retry has a + // `run_results.json` naming only the nodes that retry redid. The manifest is + // unaffected — it is a function of the project, not of what ran — so the + // state is published without results rather than with a set describing some + // other slice of the build. + results_are_partial: bool, + conn: &Connection, +) -> error::Result<()> { + let Connection::Sql(db) = conn else { + // An agent worker reaches the database only through the API, which does + // not expose this table. + return Ok(()); + }; + if p.script_path.is_empty() { + // A preview has no path to key state on, and an empty one would be + // shared by every dbt script in the workspace. + return Ok(()); + } + if p.templated_location { + // Refused on this side too, not only where a deferral reads. A template + // renders to one location per environment while the key sees the + // template, so publishing would file this run's manifest under a key a + // literal profile shares — and de-templating later would make that stale + // manifest readable as the new location's. + return Ok(()); + } + let artifacts = p.project_dir.join(ARTIFACTS_DIR); + // The manifest is what a deferral resolves through, so there is no state + // without one. Every engine writes it beside the results of a build, so this + // is the invocation that built nothing rather than a case to report. + let Ok(manifest) = tokio::fs::read_to_string(artifacts.join("manifest.json")).await else { + return Ok(()); + }; + let run_results = match results_are_partial { + true => None, + false => tokio::fs::read_to_string(artifacts.join("run_results.json")) + .await + .ok(), + }; + let environment = environment(p); + // Uploaded BEFORE the transaction, and to this publication's own keys, so two + // publishers cannot collide on them and nothing here can overwrite an + // artifact a committed row still names. A failure below has only its own + // objects to drop. + let nonce = Uuid::new_v4(); + let (manifest, manifest_key) = store( + manifest, + "manifest.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await?; + let (run_results, run_results_key) = match run_results { + Some(r) => match store( + r, + "run_results.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await + { + Ok(stored) => stored, + Err(e) => { + forget_objects(&[manifest_key, None]).await; + return Err(e); + } + }, + None => (None, None), + }; + let mine = [manifest_key.clone(), run_results_key.clone()]; + // One publisher per environment at a time, so the row and the objects it + // displaces are settled by one of them at a time. An advisory lock rather + // than the row's, because the first publish of an environment has no row to + // lock and is exactly when two runs of a newly deployed script are most + // likely to race. + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + forget_objects(&mine).await; + return Err(e.into()); + } + }; + let staged = async { + sqlx::query_scalar!( + "SELECT pg_advisory_xact_lock($1)", + publication_lock(w_id, &p.script_path, &environment) + ) + .execute(&mut *tx) + .await?; + // The script row FIRST, and held, so a rename, archive or delete of this + // path either waits for this publication or is seen by it. Reading it + // unlocked leaves a window where lifecycle cleanup finds no row to clear, + // finishes, and this transaction then commits state at a path a new + // script goes on to occupy. Script row before sidecar is also the order + // every other dbt writer takes, which is what keeps the two off a + // deadlock. + // + // The version, not just the path: "some live dbt script is here" is also + // satisfied by a script created at a path this one was renamed away from. + // A preview names no version, so `script_hash` is NULL and nothing + // matches — right for a run of content that was never deployed. + let owns_path = sqlx::query_scalar!( + "SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false AND language = 'dbt' + AND (hash = $3 OR $3 = ANY(parent_hashes)) + FOR SHARE", + w_id, + &p.script_path, + script_hash, + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + if !owns_path { + return error::Result::Ok(None); + } + // What the row points at NOW, so those objects can go once this one is + // committed in their place — never before, since a reader that has + // already read the row is about to fetch them. + let displaced = sqlx::query!( + "SELECT manifest_key, run_results_key FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(&mut *tx) + .await? + .map(|r| [r.manifest_key, r.run_results_key]) + .unwrap_or_default() + // Never a key this publication is about to commit. The keys carry a + // per-execution nonce so the two cannot coincide, and this is what says + // so rather than leaving it to be re-derived. + .map(|k| k.filter(|k| !mine.iter().flatten().any(|m| m == k))); + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest, manifest_key, run_results, + run_results_key, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now()) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET + job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest, + manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results, + run_results_key = EXCLUDED.run_results_key, updated_at = now()", + w_id, + &p.script_path, + environment, + job_id, + manifest, + manifest_key, + run_results, + run_results_key, + ) + .execute(&mut *tx) + .await?; + error::Result::Ok(Some(displaced)) + } + .await; + let displaced = match staged { + // Refused by the guard, or the write failed: nothing is committed and + // what was uploaded above has no row naming it. + Ok(None) | Err(_) => { + forget_objects(&mine).await; + return staged.map(|_| ()); + } + Ok(Some(displaced)) => displaced, + }; + // A commit that reports an error may still have committed — what was lost can + // be the acknowledgement. Dropping this run's objects would then leave the + // committed row naming objects that are gone, and every deferral would fail + // until the next publication; an orphan costs storage instead. + tx.commit().await?; + forget_objects(&displaced).await; + Ok(()) +} + +/// The environment's state, or `None` where nothing has published one. +pub(crate) async fn load( + p: &PreparedProject, + w_id: &str, + conn: &Connection, +) -> error::Result> { + let Connection::Sql(db) = conn else { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the dbt state stored for this environment, which \ + an agent worker cannot read: it reaches the database only through the API. Run this \ + script on a worker of the main group, or without `defer`" + .to_string(), + )); + }; + let environment = environment(p); + // A publication committing between the row and the objects it named drops + // those objects, so a miss is re-read rather than reported as a state that is + // not there. Re-read for as long as the row keeps MOVING: a reader takes no + // lock, so back-to-back publications can each overtake it, and a fixed one + // retry would report the second as missing. An unmoved row is the other + // answer — nothing republished, so the object really is gone. + let mut tried: Option<(Uuid, Option, Option)> = None; + for _ in 0..PUBLICATIONS_OUTRUN { + let Some(row) = sqlx::query!( + "SELECT job_id, manifest, manifest_key, run_results, run_results_key + FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(db) + .await? + else { + return Ok(None); + }; + let seen = ( + row.job_id, + row.manifest_key.clone(), + row.run_results_key.clone(), + ); + let fetched = async { + let manifest = fetch(row.manifest, row.manifest_key).await?; + let run_results = fetch(row.run_results, row.run_results_key).await?; + error::Result::Ok((manifest, run_results)) + } + .await; + match fetched { + Ok((Some(manifest), run_results)) => { + return Ok(Some(StoredState { manifest, run_results, job_id: seen.0 })) + } + Ok((None, _)) => return Ok(None), + Err(e) => { + if tried.as_ref() == Some(&seen) { + return Err(e); + } + tried = Some(seen); + } + } + } + Err(Error::internal_err( + "the dbt state for this environment was replaced faster than it could be read; run this \ + script again" + .to_string(), + )) +} + +/// How many publications a read may lose to before it gives up. Each one costs a +/// re-read, and a project publishing this often while another run defers is +/// already contending for the same relations. +const PUBLICATIONS_OUTRUN: usize = 5; + +/// A `manifest.json` for a state directory, whichever side it comes from. +/// +/// One enum because the three restores — a deferral's stored state, a retry's +/// worker-local generation, a retry's database row — differ only in where the +/// bytes are, and a second copy of the directory layout is a second chance for +/// one of them to write a directory dbt reads differently. +pub(crate) enum StateManifest { + Bytes(String), + /// A file on this worker, copied rather than read into memory: a manifest + /// grows with the project. + CopyOf(PathBuf), + None, +} + +/// Write the artifacts a dbt state directory holds, creating it if needed. +/// +/// Returns whether a `manifest.json` ended up there — a worker-local generation +/// can be pruned out from under a restore, and the caller then owes a +/// `dbt parse` for one. +pub(crate) async fn write_state_dir( + dir: &Path, + run_results: Option<&str>, + manifest: StateManifest, +) -> error::Result { + tokio::fs::create_dir_all(dir) + .await + .map_err(|e| Error::internal_err(format!("preparing the dbt state directory: {e}")))?; + if let Some(run_results) = run_results { + tokio::fs::write(dir.join("run_results.json"), run_results) + .await + .map_err(|e| Error::internal_err(format!("writing run_results.json: {e}")))?; + } + Ok(match manifest { + StateManifest::Bytes(m) => { + tokio::fs::write(dir.join("manifest.json"), m) + .await + .map_err(|e| Error::internal_err(format!("writing manifest.json: {e}")))?; + true + } + StateManifest::CopyOf(from) => tokio::fs::copy(from, dir.join("manifest.json")) + .await + .is_ok(), + StateManifest::None => false, + }) +} + +/// The advisory lock one environment's publishers take, so only one of them +/// settles the row and the objects it displaces at a time. +/// +/// Derived from the same three components as the row's key. Two environments +/// whose digests collide wait for each other, which costs a moment and nothing +/// else. +fn publication_lock(w_id: &str, script_path: &str, environment: &str) -> i64 { + let d = digest(&format!("{w_id}|{script_path}|{environment}")); + // Parsed unsigned and reinterpreted: half of all digests set the top bit, + // and read as `i64` those overflow and would collapse onto one key. + u64::from_str_radix(&d[..16], 16).unwrap_or_default() as i64 +} + +/// The object-storage key an artifact takes. +/// +/// One key per PUBLICATION, so an upload never overwrites an artifact the +/// committed row still names: a run that fails between its two uploads, or +/// between them and its row, leaves the state pointing at the pair it already +/// had. The row switches to these in one statement and the objects it displaced +/// are dropped afterwards. The path and environment are only a prefix — the row +/// is what says where an artifact is, so state that moves with a renamed script +/// keeps naming objects under the old one. Digested because a Windmill path and a +/// schema name may both carry characters an object key gives meaning to. +/// +/// The `nonce` is per EXECUTION rather than per job, because zombie recovery +/// re-runs a job under its own id: keyed on that alone, the second attempt would +/// overwrite the objects the first attempt's committed row still names, and then +/// read those same keys back as displaced and drop them. +fn object_key( + w_id: &str, + script_path: &str, + environment: &str, + job_id: &Uuid, + nonce: &Uuid, + artifact: &str, +) -> String { + format!( + "wmill_dbt_state/{w_id}/{}/{job_id}.{nonce}/{artifact}", + digest(&format!("{script_path}|{environment}")) + ) +} + +/// Put an artifact where its size says it belongs: `(inline, key)`, exactly one +/// of which is set. +#[allow(clippy::too_many_arguments)] +async fn store( + value: String, + artifact: &str, + environment: &str, + script_path: &str, + w_id: &str, + job_id: &Uuid, + nonce: &Uuid, +) -> error::Result<(Option, Option)> { + if value.len() <= *DBT_STATE_INLINE_MAX_BYTES { + return Ok((Some(value), None)); + } + let key = object_key(w_id, script_path, environment, job_id, nonce, artifact); + let size = value.len(); + if put_object(&key, value).await? { + return Ok((None, Some(key))); + } + Err(Error::BadRequest(format!( + "this project's {artifact} is {}, past the {} this instance keeps in the database, and \ + this instance has no object storage configured to hold it. Configure instance object \ + storage, or raise DBT_STATE_INLINE_MAX_BYTES", + mib(size), + mib(*DBT_STATE_INLINE_MAX_BYTES), + ))) +} + +fn mib(bytes: usize) -> String { + format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0)) +} + +/// Read an artifact back from whichever home the row names. +async fn fetch(inline: Option, key: Option) -> error::Result> { + match (inline, key) { + (Some(inline), _) => Ok(Some(inline)), + (None, Some(key)) => get_object(&key).await.map(Some), + (None, None) => Ok(None), + } +} + +/// Drop the objects nothing points at any more. Best-effort: an object left +/// behind costs storage, and there is nothing useful to do about it in the path +/// of a run that has already finished. +async fn forget_objects(keys: &[Option; 2]) { + for key in keys.iter().flatten() { + delete_object(key).await; + } +} + +/// Whether the artifact was stored. `false` means this instance has no object +/// storage to put it in. +/// +/// The INSTANCE store, where every other internal worker artifact lives — bun +/// bundles, python wheels, job logs, the global cache. Not the workspace's: +/// that bucket is the one workspace members read and write through +/// `job_helpers/*` and `wmill.write_s3_file`, so a manifest there is one any +/// member could replace, and the next deferring run would hand dbt an +/// attacker-chosen `defer_relation` for every unbuilt `ref()` while holding the +/// script's warehouse credentials. Its compiled SQL would be readable there too, +/// for a project the reader may have no access to. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn put_object(key: &str, value: String) -> error::Result { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return Ok(false); + }; + store + .put(&ObjectPath::from(key), bytes::Bytes::from(value).into()) + .await + .map_err(|e| Error::internal_err(format!("storing the dbt state at {key}: {e:#}")))?; + Ok(true) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn get_object(key: &str) -> error::Result { + let Some(store) = windmill_object_store::get_object_store().await else { + return Err(missing_storage()); + }; + let bytes = windmill_object_store::attempt_fetch_bytes(store, key).await?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::internal_err(format!("the stored dbt state is not valid UTF-8: {e}"))) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn delete_object(key: &str) { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return; + }; + if let Err(e) = store.delete(&ObjectPath::from(key)).await { + tracing::warn!("dbt: could not drop the superseded state object {key}: {e:#}"); + } +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn delete_object(_key: &str) {} + +/// A build without the instance store carries no client at all, so an oversized +/// artifact has nowhere but the row, and a row naming a key was written by a +/// worker that did have one. +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn put_object(_key: &str, _value: String) -> error::Result { + Ok(false) +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn get_object(_key: &str) -> error::Result { + Err(missing_storage()) +} + +fn missing_storage() -> Error { + Error::BadRequest( + "the dbt state for this environment is in the instance's object storage, which this \ + worker cannot reach: it is no longer configured, or this worker was built without \ + object-storage support" + .to_string(), + ) +} + +/// The stored state this run resolves its unbuilt `ref()`s through, materialised +/// into the job directory at `STATE_DIR`. +#[derive(Clone, Debug)] +pub(crate) struct Deferral { + /// The run that published the state, so the job log and the result can say + /// what this one deferred to. + pub published_by: Uuid, + /// Whether the state carries `run_results.json` beside its manifest. A build + /// recovered by node retry publishes without one, and that is the only file a + /// `result:` selector reads. + pub has_run_results: bool, +} + +/// Materialise the environment's state so `--state` has a directory to read. +/// +/// Refused rather than run without deferral where nothing is published: the run +/// would build against a `ref()` resolving into the schema it writes, and fail +/// deep inside dbt with a relation-not-found the caller has no way to connect +/// back to a missing state. +pub(crate) async fn prepare_deferral( + p: &PreparedProject, + w_id: &str, + job_dir: &str, + conn: &Connection, +) -> error::Result { + if p.script_path.is_empty() { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the state a previous run of this script \ + published, so it needs a deployed script; a preview run has no environment to have \ + published one" + .to_string(), + )); + } + // An environment is the warehouse, the target and where they RESOLVE to, and + // a `profiles.yml` that templates its schema or database resolves somewhere + // this runtime does not render. Two renderings would then share one + // environment, and a deferral after the value changed would resolve every + // unbuilt `ref()` through the previous location's manifest. + if p.templated_location { + return Err(Error::BadRequest( + "this project's profile selects its schema or database with a template, which dbt \ + renders and Windmill does not — so two environments cannot be told apart and a \ + deferral could resolve through the wrong one's manifest. Spell the target's schema \ + and database literally to use `defer`" + .to_string(), + )); + } + let Some(state) = load(p, w_id, conn).await? else { + return Err(Error::BadRequest(format!( + "no dbt state is stored for this environment ({}), so a `ref()` this run does not \ + build has no relation to resolve to. It is published by a successful run that adds \ + nothing of its own: one overriding `select` or `vars` does not publish, and neither \ + does any run of a descriptor that interpolates a `{{{{ }}}}` placeholder into `vars` \ + or a `$var:` into `env` — those describe a model set the caller's arguments decided. \ + Run this script once without `defer` and without overrides", + environment_label(p) + ))); + }; + let has_run_results = state.run_results.is_some(); + write_state_dir( + &PathBuf::from(job_dir).join(STATE_DIR), + state.run_results.as_deref(), + StateManifest::Bytes(state.manifest), + ) + .await?; + Ok(Deferral { published_by: state.job_id, has_run_results }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Every component earns its place: a deferral resolves relation NAMES, so + // state published where those names meant something else has to read as no + // state at all rather than as state that silently no longer fits. + #[test] + fn a_moved_profile_is_another_environment() { + let here = environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")); + assert_eq!( + here, + environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("other"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("dev"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("prod"), Some("marts"), Some("wh")) + ); + assert_ne!( + here, + environment_key( + Some("main"), + Some("prod"), + Some("analytics"), + Some("other_db") + ) + ); + } + + // A target name and a schema are both the user's own strings, so a component + // carrying the separator must not be able to spell another tuple's key: a + // profile moving between the two would read as the same environment and + // defer through the manifest of relations that are somewhere else. + #[test] + fn a_component_cannot_spell_another_environments_key() { + assert_ne!( + environment_key(Some("main"), Some("prod|analytics"), Some("scratch"), None), + environment_key(Some("main"), Some("prod"), Some("analytics|scratch"), None) + ); + assert_ne!( + environment_key(Some("main"), Some("prod"), Some("a"), Some("b|c")), + environment_key(Some("main"), Some("prod"), Some("a|b"), Some("c")) + ); + // A component the profile leaves out is the same environment as one it + // spells empty: there is no target named "". + assert_eq!( + environment_key(Some("main"), None, Some("a"), None), + environment_key(Some("main"), Some(""), Some("a"), Some("")) + ); + } + + // Two environments must not queue behind one advisory lock, which is what a + // digest folded through a signed parse did for every one whose top bit is + // set — half of them. + #[test] + fn each_environment_gets_its_own_publication_lock() { + let mut seen = std::collections::HashSet::new(); + for i in 0..64 { + seen.insert(publication_lock( + "ws", + "f/a/p", + &format!("main|prod|s{i}|db"), + )); + } + assert_eq!(seen.len(), 64); + assert_eq!( + publication_lock("ws", "f/a/p", "e"), + publication_lock("ws", "f/a/p", "e") + ); + } +} diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 54457f81b0..b2e69d26d6 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re // the user's own ATTACH. `// data_test` lines append verifier probes that run // against the freshly-materialized target and raise (failing the run) on // violation. Returns `None` when there is no materialize annotation or the -// target isn't a ducklake (only ducklake is materialized in v1). +// target isn't a ducklake: only ducklake has a write engine, and a `dbt://` +// target is recorded by the generic job path instead (worker.rs, +// `record_declared_warehouse_write`). fn build_materialized_query( query: &str, partition_value: Option<&str>, diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index e3e7868548..6059936af6 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -33,9 +33,11 @@ pub mod common; mod config; mod csharp_executor; +mod dbt_column_index; mod dbt_engine; mod dbt_executor; mod dbt_profiles; +mod dbt_state; #[cfg(feature = "private")] mod dedicated_worker_ee; mod dedicated_worker_oss; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7e620439b0..8dc52ef936 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -5527,7 +5527,7 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); - run_language_executor( + let result = run_language_executor( job, conn, client, @@ -5553,7 +5553,110 @@ async fn handle_code_execution_job( false, in_pipeline, ) - .await + .await; + record_declared_warehouse_write(job, conn, code, &result).await; + result +} + +/// Record the outcome of a `// materialize manual dbt:////` +/// declaration, the way the DuckDB executor records a DuckLake target. +/// +/// Nothing generates warehouse DDL, so the script issues its own write and this +/// is the only thing that turns it into a `materialized_partition` row — the +/// relation's last writer on the run page and the graph. Language-agnostic on +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. +/// +/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` +/// rows, not from this one, so a lost row costs the relation its last writer and +/// nothing else. It must not fail a job whose write already landed. +/// +/// Shares the reach of every other runtime pipeline annotation, which is this +/// function's caller: a job handed to a dedicated worker or a flow runner never +/// passes through it, so — exactly as `// partitioned` is not resolved there — +/// such a run performs its write and records no row. +async fn record_declared_warehouse_write( + job: &MiniPulledJob, + conn: &Connection, + code: &str, + result: &error::Result>, +) { + use windmill_common::materialization::{ + MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED, + }; + // A DEPLOYED script only. The annotation is a deploy-time contract — `manual`, + // a three-segment relation, a configured warehouse — checked in + // `create_script_internal`, which also required write access to the path. A + // preview, hub or inline-flow body reaches this function without any of that, + // so honouring it there would let `jobs:run` alone restamp any relation's last + // writer from a script that never touched it. + if job.kind != JobKind::Script { + return; + } + // Cheap guard: the annotation scan is skipped for the overwhelming majority + // of jobs, which carry no `materialize` line at all. + if !code.contains("materialize") { + return; + } + let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code) + .materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt) + else { + return; + }; + // The slice this run wrote, resolved once upstream (`resolve_partition_for_job`) + // and carried in the args the cascade reads too, so a partitioned producer + // records the same identity everything else propagates. + let partition = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or_else(|| UNPARTITIONED.to_string()); + let (status, error) = match result { + Ok(_) => (MaterializationStatus::Materialized, None), + Err(e) => (MaterializationStatus::Failed, Some(e.to_string())), + }; + let recorded = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + &job.workspace_id, + windmill_common::assets::AssetKind::Dbt, + &m.target_path, + &partition, + status, + None, + None, + Some(job.id), + error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http( + client, + &job.workspace_id, + &RecordMaterializationRequest { + asset_kind: windmill_common::assets::AssetKind::Dbt, + asset_path: m.target_path.clone(), + partition, + status, + snapshot_id: None, + row_count: None, + job_id: Some(job.id), + error, + schema: None, + }, + ) + .await + } + }; + if let Err(e) = recorded { + tracing::warn!( + "recording the materialization of dbt://{} failed: {e:#}", + m.target_path + ); + } } /// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 12930e8de4..2899ff694c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.804.0"; +export const VERSION = "v1.805.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0f46ec76be..b35efc02a3 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + getWmillYamlPath, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; import lintCommand from "./lint.ts"; @@ -21,6 +25,7 @@ import newCommand from "./new.ts"; import generateAgentsCommand from "./generate_agents.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; export interface AppFile { @@ -420,6 +425,8 @@ async function push( if (isRawAppByName || hasRawAppYaml) { const { pushRawApp } = await import("./raw_apps.ts"); const merged = await mergeConfigWithConfigFile(opts); + // Raw-app ownership preservation is not implemented on either push + // path: sync push hands pushRawApp no context either. await pushRawApp( workspace.workspaceId, remotePath, @@ -429,7 +436,16 @@ async function push( ); log.info(colors.bold.underline.green("Raw app pushed")); } else { - await pushApp(workspace.workspaceId, remotePath, absoluteFilePath); + await pushApp( + workspace.workspaceId, + remotePath, + absoluteFilePath, + undefined, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), + ); log.info(colors.bold.underline.green("App pushed")); } } diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7a383fcfeb..863690ce7d 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -4,7 +4,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; -import { dirname, sep as SEP } from "node:path"; +import { dirname, sep as SEP, resolve as pathResolve } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; @@ -21,11 +21,16 @@ import { } from "../../core/context.ts"; import { resolve, track_job, pollForJobResult } from "../script/script.ts"; import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; -import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + SyncOptions, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { collectPathScriptPaths, replaceInlineScripts, @@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo if (!validatePath(remotePath)) { return; } + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); - await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); + await pushFlow( + workspace.workspaceId, + remotePath, + filePath, + opts.message, + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) + ); log.info(colors.bold.underline.green("Flow pushed")); } diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index 2eb6ba42b8..096615c757 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -6,13 +6,19 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import * as wmill from "../../../gen/services.gen.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { lookupUsernameByEmail } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + lookupUsernameByEmail, +} from "../../core/permissioned_as.ts"; import { GlobalOptions, @@ -299,8 +305,12 @@ async function disable(opts: GlobalOptions, path: string) { } async function push(opts: GlobalOptions, filePath: string, remotePath: string) { + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); if (!validatePath(remotePath)) { return; @@ -317,7 +327,8 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { workspace.workspaceId, remotePath, undefined, - parseFromFile(filePath) + parseFromFile(filePath), + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) ); console.log(colors.bold.underline.green("Schedule pushed")); } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index fada434e0d..d2afdeeee3 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -7,6 +7,7 @@ import { validatePath, } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import { writeFile, stat, mkdir } from "node:fs/promises"; import { Buffer } from "node:buffer"; @@ -58,6 +59,7 @@ import { SyncOptions, mergeConfigWithConfigFile, readConfigFile, + readEffectiveSyncBehavior, } from "../../core/conf.ts"; import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; @@ -231,7 +233,11 @@ async function push(opts: PushOptions, filePath: string) { opts.message, opts, await getRawWorkspaceDependencies(true), - codebases + codebases, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace) + ) ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); } diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2702e13714..d75dc81912 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -104,7 +104,7 @@ export async function downloadZip( // from v1 the on-behalf-of address is stripped below, so the tarball sends the // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs - // on flow / script / app rows. Default-off on the server protects cross- + // on script / flow / app / variable rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities // missing in the target workspace; the CLI sync flow explicitly wants them. const baseParams = `&plain_secret=${plainSecrets ?? false @@ -150,7 +150,18 @@ export async function downloadZip( } if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + log.info( + colors.red( + `Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` + + (workspace.name !== workspace.workspaceId + ? ` (resolved from profile '${workspace.name}')` + : "") + + `.\n` + + `Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` + + ` - check 'wmill workspace list' (the 'workspace id' column)\n` + + ` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)` + ) + ); } else { log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); if (body) log.info(colors.red(body)); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index be5d9e5639..99a098f848 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -76,7 +76,8 @@ import { } from "../../utils/utils.ts"; import { getEffectiveSettings, - getWorkspaceNames, + inferWsNameFromProfile, + resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, SyncOptions, @@ -85,7 +86,10 @@ import { WorkspaceEntryConfig, } from "../../core/conf.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { preCheckPermissionedAs } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + preCheckPermissionedAs, +} from "../../core/permissioned_as.ts"; import { fromWorkspaceSpecificPath, toWorkspaceSpecificPath, @@ -429,37 +433,6 @@ export function computeWsSpecificFlagOnlyPushes( return out; } -// Resolve workspace name from a --branch override (git branch → workspace name). -// Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch( - opts: SyncOptions, - branchName: string, -): string { - const match = findWorkspaceByGitBranch(opts.workspaces, branchName); - return match ? match[0] : branchName; -} - -// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, -// then --workspace → matching config key (incl. when --base-url is set). Returns -// undefined when no flag-based resolution applies; callers then fall back to -// inferWsNameFromProfile on the resolved workspace profile. -export function resolveWsNameForConfigFromFlags( - opts: SyncOptions & { branch?: string; workspace?: string }, -): string | undefined { - if (opts.branch) { - return resolveWsNameFromBranch(opts, opts.branch); - } - if (opts.workspace) { - // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, - // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. - const validKeys = getWorkspaceNames(opts.workspaces); - if (validKeys.includes(opts.workspace)) { - return opts.workspace; - } - } - return undefined; -} - // Warn if --workspace overrides auto-detected branch or if workspace not in config. function warnWorkspaceOverride( opts: SyncOptions, @@ -507,33 +480,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { return wsName; } -// After resolveWorkspace, infer the workspace config name from the resolved profile -// by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile( - opts: SyncOptions, - profile: { remote: string; workspaceId: string }, -): string | undefined { - if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter( - (k) => k !== "commonSpecificItems", - ); - for (const name of wsNames) { - const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; - if (!entry?.baseUrl) continue; - try { - const entryUrl = new URL(entry.baseUrl).toString(); - const profileUrl = new URL(profile.remote).toString(); - const entryWsId = entry.workspaceId ?? name; - if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { - return name; - } - } catch { - continue; - } - } - return undefined; -} - // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< T extends GlobalOptions & SyncOptions & { repository?: string }, @@ -5540,27 +5486,19 @@ export async function push( return; } - let permissionedAsContext: PermissionedAsContext | undefined = undefined; - if (parseSyncBehavior(opts.syncBehavior) >= 1) { - const user = await wmill.whoami({ workspace: workspace.workspaceId }); - const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + const permissionedAsContext: PermissionedAsContext | undefined = + await buildPermissionedAsContext( + workspace.workspaceId, + opts.syncBehavior, ); - permissionedAsContext = { - userCache: new Map(), - userIsAdminOrDeployer, - userEmail: user.email, - }; - + if (permissionedAsContext) { // ws_specific_flag changes have no content payload, so they don't // affect permissioned_as resolution — filter them out before the // pre-check (which expects only added/edited/deleted). await preCheckPermissionedAs( changes.filter((c) => c.name !== "ws_specific_flag"), - user.email, - userIsAdminOrDeployer, + permissionedAsContext.userEmail, + permissionedAsContext.userIsAdminOrDeployer, opts.acceptOverridingPermissionedAsWithSelf ?? false, !!process.stdin.isTTY, ); diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 83b62f9eb4..b011a0f0b7 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -23,7 +23,7 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { GlobalOptions, isSuperset, @@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; +import { readEffectiveSyncBehavior } from "../../core/conf.ts"; type Trigger = { http: HttpTrigger; @@ -620,8 +622,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise; } /** @@ -152,37 +154,42 @@ export async function pushVariable( log.debug(`Variable ${remotePath} does not exist on remote`); } + // extra_perms is synced independently via /acls/* (see applyExtraPermsDiff) + // so a perm-only edit never rewrites the variable value. Strip the field from + // the body that goes to update_variable / create_variable and treat it as a + // separate step both for the up-to-date short-circuit and after the write. + const { extra_perms: localPerms, ...localVariableBody } = localVariable; + if (variable) { - if (isSuperset(localVariable, variable)) { + if (isSuperset(localVariableBody, variable)) { log.debug(`Variable ${remotePath} is up-to-date`); - return; - } + } else { + log.debug(`Variable ${remotePath} is not up-to-date, updating`); - log.debug(`Variable ${remotePath} is not up-to-date, updating`); - - // Apply is_secret only when it differs from the remote (the value is always - // sent, so the server allows the flag change). Upgrades (non-secret->secret) - // always apply; downgrades only when explicitly allowed (single-file push) — - // see allowSecretDowngrade. `undefined` leaves the flag untouched. - let nextIsSecret: boolean | undefined = undefined; - if (localVariable.is_secret !== variable.is_secret) { - if (localVariable.is_secret) { - nextIsSecret = true; - } else if (allowSecretDowngrade) { - nextIsSecret = false; + // Apply is_secret only when it differs from the remote (the value is always + // sent, so the server allows the flag change). Upgrades (non-secret->secret) + // always apply; downgrades only when explicitly allowed (single-file push) — + // see allowSecretDowngrade. `undefined` leaves the flag untouched. + let nextIsSecret: boolean | undefined = undefined; + if (localVariableBody.is_secret !== variable.is_secret) { + if (localVariableBody.is_secret) { + nextIsSecret = true; + } else if (allowSecretDowngrade) { + nextIsSecret = false; + } } - } - await wmill.updateVariable({ - workspace, - path: remotePath.replaceAll(SEP, "/"), - alreadyEncrypted: !plainSecrets, - requestBody: { - ...localVariable, - is_secret: nextIsSecret, - ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), - }, - }); + await wmill.updateVariable({ + workspace, + path: remotePath.replaceAll(SEP, "/"), + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariableBody, + is_secret: nextIsSecret, + ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), + }, + }); + } } else { log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await wmill.createVariable({ @@ -190,11 +197,22 @@ export async function pushVariable( alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath.replaceAll(SEP, "/"), - ...localVariable, + ...localVariableBody, ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), }, }); } + + // Synced whether or not the body changed. No refetch: folder perms are never + // merged onto item.extra_perms, and the update/create body carries no + // extra_perms, so the value getVariable read above is still the remote one. + await applyExtraPermsDiff( + workspace, + "variable", + remotePath.replaceAll(SEP, "/"), + localPerms, + (variable as any)?.extra_perms, + ); } async function push( diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1fcce7a43f..0c746d2261 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null { } export const GLOBAL_CONFIG_OPT = { noCdToRoot: false }; -function findWmillYaml(): string | null { + +// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir. +function locateWmillYaml(): string | null { const startDir = resolve(process.cwd()); const isInGitRepo = isGitRepository(); const gitRoot = isInGitRepo ? getGitRepoRoot() : null; @@ -176,6 +178,13 @@ function findWmillYaml(): string | null { currentDir = parentDir; } + return foundPath; +} + +function findWmillYaml(): string | null { + const startDir = resolve(process.cwd()); + const foundPath = locateWmillYaml(); + // If wmill.yaml was found in a parent directory, warn the user and change working directory if ( !GLOBAL_CONFIG_OPT.noCdToRoot && @@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } +/** + * Look up one `workspaces` entry, for diagnostics only. readConfigFile() must + * not be used for that: it chdirs to the config's directory, exits on an + * unsupported syncBehavior and throws on a malformed file. A diagnostic may + * never fail or relocate the command it is diagnosing. + */ +export async function peekWorkspaceEntry( + workspaceName: string +): Promise { + if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) { + return undefined; + } + const wmillYamlPath = locateWmillYaml(); + if (!wmillYamlPath) { + return undefined; + } + try { + const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions; + const workspaces = + conf?.workspaces ?? + conf?.gitBranches ?? + conf?.environments ?? + conf?.git_branches; + const entry = (workspaces as any)?.[workspaceName]; + return typeof entry === "object" && entry !== null ? entry : undefined; + } catch (e) { + log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`); + return undefined; + } +} + let legacyConfigWarned = false; export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { @@ -630,6 +670,83 @@ export async function getEffectiveSettings( return effective; } +// Resolve workspace name from a --branch override (git branch → workspace name). +// Falls back to using the branch value as-is (backward compat: old key = branch name). +function resolveWsNameFromBranch(opts: SyncOptions, branchName: string): string { + const match = findWorkspaceByGitBranch(opts.workspaces, branchName); + return match ? match[0] : branchName; +} + +// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, +// then --workspace → matching config key (incl. when --base-url is set). Returns +// undefined when no flag-based resolution applies; callers then fall back to +// inferWsNameFromProfile on the resolved workspace profile. +export function resolveWsNameForConfigFromFlags( + opts: SyncOptions & { branch?: string; workspace?: string } +): string | undefined { + if (opts.branch) { + return resolveWsNameFromBranch(opts, opts.branch); + } + if (opts.workspace) { + // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, + // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. + const validKeys = getWorkspaceNames(opts.workspaces); + if (validKeys.includes(opts.workspace)) { + return opts.workspace; + } + } + return undefined; +} + +/** + * Match a workspace config entry to a resolved workspace profile by remote + + * workspace id. The fallback for when no flag names the entry outright. + */ +export function inferWsNameFromProfile( + opts: SyncOptions, + profile: { remote: string; workspaceId: string } +): string | undefined { + if (!opts.workspaces) return undefined; + for (const name of getWorkspaceNames(opts.workspaces)) { + const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; + if (!entry?.baseUrl) continue; + try { + const entryUrl = new URL(entry.baseUrl).toString(); + const profileUrl = new URL(profile.remote).toString(); + const entryWsId = entry.workspaceId ?? name; + if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { + return name; + } + } catch { + continue; + } + } + return undefined; +} + +/** + * `syncBehavior` as the workspace being pushed to sees it. The top level alone + * misses a `workspaces..overrides.syncBehavior`, which is where a repo + * that varies settings per workspace puts it, and the entry to read is the one + * `--workspace` names — falling back to the profile, then to the git branch — + * the same order `sync push` resolves it in. + */ +export async function readEffectiveSyncBehavior( + opts: { workspace?: string }, + profile?: { remote: string; workspaceId: string } +): Promise { + const config = await readConfigFile({ warnIfMissing: false }); + const named = resolveWsNameForConfigFromFlags({ ...config, ...opts }); + const effective = await getEffectiveSettings( + config, + undefined, + false, + true, + named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined) + ); + return effective.syncBehavior; +} + const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]); /** diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 35bd6bc2e7..b5494b112c 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.804.0"; +export const VERSION = "1.805.0"; diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 502157f0a7..80b27ea52e 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -20,6 +20,7 @@ import { import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts"; import { readConfigFile, + peekWorkspaceEntry, findWorkspaceByGitBranch, getEffectiveWorkspaceId, getWmillYamlPath, @@ -219,6 +220,9 @@ async function tryResolveWorkspace( // First try: look up workspace by name in wmill.yaml workspaces config const config = await readConfigFile({ warnIfMissing: false }); const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined; + // What wmill.yaml said to target, kept for the fallback below: a profile + // found by name can silently point somewhere else entirely. + let configuredTarget: { workspaceId: string; baseUrl: string } | undefined; if (wsEntry?.baseUrl) { const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry); let normalizedBaseUrl: string; @@ -231,6 +235,8 @@ async function tryResolveWorkspace( }; } + configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl }; + // Find matching profile by baseUrl + workspaceId const allProfs = await allWorkspaces(opts.configDir); const matching = allProfs.filter( @@ -283,6 +289,22 @@ async function tryResolveWorkspace( ), }; } + if ( + configuredTarget && + (e.workspaceId !== configuredTarget.workspaceId || + e.remote !== configuredTarget.baseUrl) + ) { + log.warnStderr( + colors.yellow( + `⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` + + ` Run: wmill workspace add ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}` + ) + ); + } + log.infoStderr( + `Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}` + ); (opts as any).__secret_workspace = e; return { isError: false, value: e }; } @@ -486,6 +508,8 @@ export async function resolveWorkspace( return process.exit(-1); } + let resolved: Workspace | undefined; + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { let existingWorkspace = await getWorkspaceByName( @@ -523,19 +547,45 @@ export async function resolveWorkspace( ); return process.exit(-1); } - return { + resolved = { ...existingWorkspace, token: opts.token, }; } } - return { + resolved ??= { remote: normalizedBaseUrl, workspaceId: opts.workspace, name: opts.workspace, token: opts.token, }; + + // --base-url pins the target, so wmill.yaml's `workspaces` block is never + // consulted and `--workspace` reaches the API as a workspace id. Name the + // id being sent, and the mapping being skipped, before the request 404s + // on an id the user never typed. + // Only an explicit `workspaceId:` is worth reporting: an entry without one + // maps the name to itself, leaving nothing to correct. + const yamlEntry = await peekWorkspaceEntry(opts.workspace); + const yamlWorkspaceId = yamlEntry?.workspaceId; + if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) { + log.warnStderr( + colors.yellow( + `⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` + + ` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.` + ) + ); + } + log.infoStderr( + `Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` + + (resolved.name !== resolved.workspaceId + ? `, profile '${resolved.name}')` + : ")") + ); + (opts as any).__secret_workspace = resolved; + return resolved; } else { log.infoStderr( colors.red( diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 5ac48753d6..57570b3549 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,7 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { userCache: Map; @@ -10,6 +11,35 @@ export interface PermissionedAsContext { userEmail: string; } +/** + * The whole-tree `sync push` and the single-item `push` commands must resolve + * ownership the same way, so both build the context here: a push that leaves it + * undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it. + * Undefined below syncBehavior v1, where that reassignment is the contract, and + * for a caller who is neither admin nor in `wm_deployers` the backend enforces + * it anyway — the flag on the context is what keeps the CLI from claiming + * otherwise. + */ +export async function buildPermissionedAsContext( + workspace: string, + syncBehavior: string | number | undefined +): Promise { + if (parseSyncBehavior(syncBehavior) < 1) { + return undefined; + } + const user = await wmill.whoami({ workspace }); + const userIsAdminOrDeployer = + user.is_admin || (user.groups ?? []).includes("wm_deployers"); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + ); + return { + userCache: new Map(), + userIsAdminOrDeployer, + userEmail: user.email, + }; +} + async function ensureUserCache( workspace: string, cache: Map diff --git a/cli/test/base_url_workspace_resolution_unit.test.ts b/cli/test/base_url_workspace_resolution_unit.test.ts new file mode 100644 index 0000000000..66f618d113 --- /dev/null +++ b/cli/test/base_url_workspace_resolution_unit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { resolveWorkspace } from "../src/core/context.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; + +const BASE_URL = "http://localhost:9999/"; + +// --base-url pins the target: --workspace reaches the API as a workspace id and +// wmill.yaml is not consulted. The warning that says so may only peek at the +// file — readConfigFile() exits on an unsupported syncBehavior and throws on a +// malformed one, so resolving through it lets an unrelated config fail a +// command that never needed it. +async function withWmillYaml( + wmillYaml: string, + fn: (opts: GlobalOptions) => Promise +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_")); + const originalCwd = process.cwd(); + try { + await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml); + await writeFile(await getWorkspaceConfigFilePath(configDir), ""); + + process.chdir(repoDir); + await fn({ + configDir, + baseUrl: BASE_URL, + token: "sometoken", + workspace: "staging", + } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +describe("--base-url workspace resolution", () => { + const rejectedConfigs: [string, string][] = [ + ["an unsupported syncBehavior", "syncBehavior: v2\n"], + ["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'], + ]; + + for (const [label, wmillYaml] of rejectedConfigs) { + test(`resolves despite ${label}`, async () => { + await withWmillYaml(wmillYaml, async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + }); + }); + } + + test("a workspaces mapping never overrides the explicit workspace id", async () => { + await withWmillYaml( + "workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n", + async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + } + ); + }); +}); diff --git a/cli/test/schedule_push_permissioned_as_unit.test.ts b/cli/test/schedule_push_permissioned_as_unit.test.ts new file mode 100644 index 0000000000..941415409d --- /dev/null +++ b/cli/test/schedule_push_permissioned_as_unit.test.ts @@ -0,0 +1,125 @@ +/** + * Regression guard: the standalone `wmill schedule push` must resolve ownership + * the same way `wmill sync push` does. It only preserves the remote's + * `permissioned_as` when the command hands `pushSchedule` a context, so a push + * that builds none silently reassigns the schedule to whoever ran it. + */ + +import { expect, test, describe, beforeEach, mock } from "bun:test"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let updateScheduleCalls: any[] = []; +let remotePermissionedAs: string | undefined = "u/svc"; + +const REMOTE_SCHEDULE = () => ({ + path: "u/admin/sched", + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: "u/admin/script", + is_flow: false, + args: {}, + enabled: false, + summary: "before", + permissioned_as: remotePermissionedAs, +}); + +mock.module("../gen/services.gen.ts", () => ({ + getSchedule: async () => REMOTE_SCHEDULE(), + updateSchedule: async (a: unknown) => { + updateScheduleCalls.push(a); + }, + whoami: async () => ({ + email: "deployer@windmill.dev", + username: "deployer", + is_admin: true, + groups: [], + }), +})); + +const realContext = await import("../src/core/context.ts"); +mock.module("../src/core/context.ts", () => ({ + ...realContext, + resolveWorkspace: async () => ({ + workspaceId: "w", + name: "w", + remote: "http://localhost/", + token: "t", + }), +})); + +const realAuth = await import("../src/core/auth.ts"); +mock.module("../src/core/auth.ts", () => ({ + ...realAuth, + requireLogin: async () => ({}), +})); + +const scheduleCommand = (await import("../src/commands/schedule/schedule.ts")) + .default; + +async function pushIn(wmillYamlTail: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_")); + await writeFile( + join(dir, "wmill.yaml"), + `defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`, + "utf-8" + ); + await writeFile( + join(dir, "sched.schedule.yaml"), + `schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`, + "utf-8" + ); + + const cwd = process.cwd(); + process.chdir(dir); + try { + await scheduleCommand.parse([ + "push", + "sched.schedule.yaml", + "u/admin/sched", + ]); + } finally { + process.chdir(cwd); + } +} + +describe("wmill schedule push ownership", () => { + beforeEach(() => { + updateScheduleCalls = []; + remotePermissionedAs = "u/svc"; + }); + + test("keeps the remote's permissioned_as under syncBehavior v1", async () => { + await pushIn("syncBehavior: v1\n"); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.summary).toBe("after"); + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + // The entry to read is the one matching the workspace being pushed to, not + // the top level: a repo that varies settings per workspace puts syncBehavior + // under `overrides` and nowhere else. + test("reads syncBehavior from the target workspace's overrides", async () => { + await pushIn( + `workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n` + ); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + test("leaves ownership to the backend below syncBehavior v1", async () => { + await pushIn(""); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBeUndefined(); + expect(body.preserve_permissioned_as).toBeUndefined(); + }); +}); diff --git a/cli/test/variable_resource_push.test.ts b/cli/test/variable_resource_push.test.ts index e6bc84c326..b77ed48b50 100644 --- a/cli/test/variable_resource_push.test.ts +++ b/cli/test/variable_resource_push.test.ts @@ -361,6 +361,126 @@ describe("variable", () => { expect(content).toContain("is_secret: false"); }); }); + + test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/perms_var_${uniqueId}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: varPath, + value: "perms_test_value", + is_secret: false, + description: "Variable for extra_perms test", + }), + } + ); + expect(createResp.status).toBeLessThan(300); + await createResp.text(); + + const aclResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/acls/add/variable/${varPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ owner: "g/all", write: true }), + } + ); + expect(aclResp.status).toBeLessThan(300); + await aclResp.text(); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`, + "utf-8" + ); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir + ); + expect(pullResult.code).toEqual(0); + + const localPath = join(tempDir, `${varPath}.variable.yaml`); + const pulled = await readFile(localPath, "utf-8"); + expect(pulled).toContain("extra_perms:"); + expect(pulled).toContain("g/all: true"); + + const beforeResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const before = await beforeResp.json(); + + // Perm-only edit: downgrade the grant to read. + await writeFile( + localPath, + pulled.replace("g/all: true", "g/all: false"), + "utf-8" + ); + + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + const afterResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const after = await afterResp.json(); + expect(after.extra_perms).toEqual({ "g/all": false }); + // Routed through /acls/* rather than update_variable, so the row itself + // is untouched. + expect(after.edited_at).toEqual(before.edited_at); + expect(after.value).toEqual("perms_test_value"); + + // A yaml with no extra_perms field at all is "no opinion": a checkout + // that predates ACL sync must never revoke UI-managed grants. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`, + "utf-8" + ); + const noOpinionResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(noOpinionResult.code).toEqual(0); + + const noOpinionApiResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + expect((await noOpinionApiResp.json()).extra_perms).toEqual({ + "g/all": false, + }); + + // An owner present remotely but absent from a *present* map is revoked — + // the one direction that can destroy a grant. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`, + "utf-8" + ); + const revokeResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(revokeResult.code).toEqual(0); + + const finalResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const final = await finalResp.json(); + expect(final.extra_perms).toEqual({}); + }); + }); }); // ============================================================================= diff --git a/cli/test/workspace_key_filename_integration.test.ts b/cli/test/workspace_key_filename_integration.test.ts index ae22f191bf..a2e731eae5 100644 --- a/cli/test/workspace_key_filename_integration.test.ts +++ b/cli/test/workspace_key_filename_integration.test.ts @@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml"; import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts"; import { findResourceFile } from "../src/commands/script/script.ts"; -import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts"; -import type { SyncOptions } from "../src/core/conf.ts"; +import { + resolveWsNameForConfigFromFlags, + type SyncOptions, +} from "../src/core/conf.ts"; // Integration tests covering the bug where workspace-specific filenames used // the raw git branch name instead of the wmill.yaml workspace config key. diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index dc06a22fa0..926948a676 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -13,8 +13,8 @@ the dominant way dbt is orchestrated today. - **In**: run an unmodified dbt project synced into Windmill, one Windmill job per invocation, live per-model observability, dbt models as first-class assets in the existing asset graph. -- **Out**: one Windmill job per dbt model, `state:modified` / slim CI, - `dbt docs` hosting, semantic layer, dbt platform integration. +- **Out**: one Windmill job per dbt model, slim CI orchestration, `dbt docs` + hosting, semantic layer, dbt platform integration. - **CE**: the runtime, the manifest ingest, the asset graph and every piece of UI ship in CE, as do all adapters except two. Only the `mssql` and `oracle` adapters are EE, mirroring the native `ScriptLang` boundary (decision 21). @@ -35,8 +35,8 @@ the dominant way dbt is orchestrated today. | 10 | Private repo auth | Not applicable: the project is synced, not fetched | | 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | -| 13 | Manifest storage | Sidecar table for nodes/edges. Full manifest **not** stored — see below | -| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below | +| 13 | Manifest storage | Sidecar table for nodes/edges; the whole manifest is kept once per environment, for deferral — see below | +| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** and real column schemas come from the engine's parquet index, opt-in per project — see below | | 15 | Node rendering | Asset nodes per model plus one runnable node for the script | | 16 | Progress | Live, from the JSON event stream | | 17 | Test failures | Honor dbt's own `severity` | @@ -47,6 +47,8 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | +| 26 | Deferral | A durable state per environment, published by the runs whose relations are the script's; `defer` is a per-run toggle. See below | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -157,11 +159,15 @@ build and an enterprise build whose key did not verify. workspace warehouse's NAME, so two scripts running against the same warehouse agree on identity. -The SCHEME names the producer, because dbt is the only thing that creates one of -these: no other language derives warehouse relations, `// materialize` takes -DuckLake targets only, and a dbt run does not dispatch. Calling the kind -something generic promised a parity with native Snowflake and BigQuery scripts -that does not exist. +The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what +put warehouse relations in the asset graph and is what derives them from a +project; no other language *infers* one, and calling the kind something generic +promised a parity with native Snowflake and BigQuery scripts that does not exist. +A script can nonetheless DECLARE that it writes one — `// materialize manual +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -429,7 +435,9 @@ Two things make that safe rather than a widening: model set and its relations are already visible to them. - **`raw_code` is gated separately**, on an `EXISTS` against `script` in the authed transaction. The body of a model is the project's source code and stays - behind access to the project, whatever the shape query resolved. + behind access to the project, whatever the shape query resolved. `column_schema` + and the column trace behind `/jobs/dbt_column_lineage/{id}` take the same gate, + for the same reason: both are the shape of what the author wrote. The path and hash coming from the job row rather than the query also means a caller cannot pin one project's version while naming another's run. @@ -648,10 +656,13 @@ Two consequences worth knowing: dropped would be filtered out of its own run's graph. The pinned version's nodes are the scope instead. -## No cascade from dbt, and no pipeline membership +## No cascade *from* dbt, and no pipeline membership A finished dbt run does not trigger anything. Its models are recorded, drawn and -tracked; they do not fan out. +tracked; they do not fan out. The opposite direction does: a script that declares +`// materialize manual dbt:////` is an ordinary producer +of that relation, and its completion wakes `# on dbt://` subscribers +through the same fan-out every other asset kind uses. A dbt script is also not a pipeline member (`in_pipeline` is forced false for `ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like @@ -663,22 +674,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what puts a native script reading one of them on the same node, and it is independent of pipeline membership. -dbt already orders its own DAG, so a cascade would only ever add one thing: -waking a Windmill script that reads a mart. That edge is real but narrow, and -only half of it exists — nothing outside dbt can declare a `dbt://` write -(`// materialize` accepts DuckLake targets only), so the reverse direction, an -ingestion script waking a dbt project, cannot be expressed at all. - -Against that, dispatching correctly from dbt is not cheap. A run's `select` can -build any subset of the project, so the deploy-time write set is not what ran; -using it wakes consumers of relations the run never touched, and narrowing it -needs a per-job record of what was built, which the per-relation state table -cannot supply (it keeps one row per relation, stamped with the last writer). +dbt already orders its own DAG, so a cascade out of a run would only ever add one +thing: waking a Windmill script that reads a mart. That edge is real but narrow, +and dispatching it correctly is not cheap. A run's `select` can build any subset +of the project, so the deploy-time write set is not what ran; using it wakes +consumers of relations the run never touched, and narrowing it needs a per-job +record of what was built, which the per-relation state table cannot supply (it +keeps one row per relation, stamped with the last writer). So dbt materializes and reports, and `asset_dispatch` returns early for -`ScriptLang::Dbt`. A `# on dbt://` subscription is refused outright at -deploy rather than accepted and left dormant — an edge drawn on the canvas that -can never fire is worse than an error saying so. +`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should +notify — that decision is the work, not the plumbing. + +### Declaring the write, and which subscriptions are refused + +`// materialize manual dbt:////` is how an ingestion +script says it writes a warehouse relation. `manual` is not a mode but the only +mode: nothing generates warehouse DDL, so the script issues its own write and +Windmill records the outcome — the same `materialized_partition` row a DuckLake +target lands, so the relation carries a last writer on the run page and the graph. +It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path +(`record_declared_warehouse_write`) rather than in an executor, for the same +reason. Identity is unchanged — the physical relation — so the ingestion script +and the dbt model reading it are one node, and a `source` declared on the relation +puts the whole thing on one lineage. `// data_test` is refused beside it: those +checks are probes the DuckDB executor splices around a managed write, so on a +warehouse relation — which the script writes itself, from any language — nothing +would run them, and a declarer would deploy green with its assertions silently +skipped. Assert on the relation with a dbt test in the project that reads it. +The `` segment is resolved at +deploy for the same reason a descriptor's `profile.warehouse` is: a name no +warehouse answers to is not a namespace, it strands the write on a node nothing +else reaches. + +Known boundary, shared with every other runtime pipeline annotation: the record +is written from the normal execution path, and recording and cascading are decided +separately, so the routes off it differ. + +* A **dedicated worker** never enters that path — it bypasses the record exactly + as it bypasses `// partitioned` resolution — while its job is still a top-level + `Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It + cascades and records nothing, leaving the relation with no last writer. +* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which + `is_eligible_kind` rejects. Neither record nor cascade. +* A **flow step running a deployed script** enters the path as a `Script` job, so + it records — and carries a `flow_step_id`, so it never cascades. +* A **flow step with an inline body** is a `FlowScript` job, which the recording + guard excludes along with previews: neither. + +Fixing the recording half is one change for every runtime pipeline annotation, +not this one. + +A `# on dbt://` subscription is held to the same relation a producer +is — a whole `//` under a configured warehouse, checked +by the validator the `// materialize` target goes through, since two spellings of +that rule would refuse and accept the same string. Beyond that it is refused in +exactly one shape: when every script that writes that relation is a dbt one. Nothing +produces it yet is NOT that shape — a subscriber may be deployed before its +producer, as for every other asset kind, and refusing there would break +deploy-order-independent syncs. A dbt script may neither subscribe nor declare a +`// materialize`: its graph ingest republishes that path's trigger and asset rows +wholesale, so either annotation would deploy something the dependency job then +silently removes — while the declared write would still stamp the relation on +every run. + +The producer set is read as it stands committed, minus the deploying script's own +rows — those describe the version being replaced, so a script dropping its +`// materialize` while adding a subscription would otherwise count itself as the +producer that wakes it, which it could not be anyway (the dispatcher skips +self-loops). + +What that leaves is a subscription accepted while it was live and later orphaned. +A dbt project that claims the relation afterwards names those edges in its own log +rather than leaving them silently dormant — the same "an edge that can never fire +is worse than saying so" the refusal is for, at the other point where it is +knowable. Both points that publish ownership warn: the deploy, and a run whose +static descriptor found its profile moved. An agent run publishes none — it is +forced to per-run models, so it stores a job-pinned snapshot and leaves workspace +ownership with the deployed graph — so it cannot orphan a subscription either. + +Two orphanings are reported nowhere, and both are accepted rather than overlooked. +A native producer that drops its `// materialize` and leaves dbt alone on the +relation: the deploy that causes it does not touch the subscriber. And the +interleaving where a dbt ingest commits between a subscriber's producer check and +its own commit — the check sees no producer and accepts, the ingest's warning +query sees no trigger and says nothing. Closing the second means a per-relation +lock shared by the deploy path and the ingest, and the ingest takes +`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation +locks first inverts that order into a deadlock across two subsystems — a worse +failure than the cosmetic edge it would prevent. Both are bounded the same way: +the next deploy of that project warns, and the canvas is where they show +meanwhile. A plain READ still renders the consumer beside the model, which is what makes the lineage one graph — but it is written in the script's own code, not in a @@ -687,8 +775,8 @@ comment: the body parsers resolve an asset URI from a string literal Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the languages with a body-asset parser; the native warehouse ones (snowflake, bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart -they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a -selective run should notify — that decision is the work, not the plumbing. +they consume joins the graph only once that inference exists — while a relation +one of them WRITES joins it now, through the annotation. ## Live per-model progress, and why only dbt-core 1.x has it @@ -857,6 +945,8 @@ profile: select: ["tag:nightly+"] exclude: [] test_behavior: build # build | after_all | none +column_lineage: false # opt in to the static-analysis pass that + # produces column-level lineage (decision 14) vars: # typed: numbers/bools/lists keep their type, run_date: "{{ run_date }}" # and string leaves take job arguments strict: false @@ -1143,19 +1233,500 @@ block, since that is what `dbt_run_state` saves and `invocation_args` publishes. without failing. Overriding this would make the same project behave differently on Windmill than locally, breaking the core promise. +## Durable state per environment, and what defers to it + +`dbt --defer --state ` resolves a `ref()` the run does not build to the +relation the manifest in `` names, instead of to the schema this run writes +into. That is what lets one model be rebuilt into a scratch schema without +rebuilding everything above it, and it needs a manifest of the environment the +project actually lives in. + +Nothing that already existed could supply one. `dbt_run_state` answers a +different question — it holds the LAST run whatever its outcome, keyed by the +principal, so `dbt retry` can resume its failures — and the worker-local +generations behind it are a cache: the next run of a project usually lands on a +worker holding neither artifact. So the state is its own table, +`dbt_environment_state`, one row per (workspace, script path, environment), +holding `manifest.json` and `run_results.json` from the last SUCCESSFUL run. +Success is half of the contract: a relation a later run defers to has to exist. + +### The environment is the warehouse, the target and where they resolve to + +The workspace warehouse's name, the target dbt actually runs, and the database +and schema that target resolves to — the pair `relation_root` reports to the +graph's drift check. Each component is length-prefixed rather than joined on a +separator — `|||`, each written `:`, +so `main`/`prod`/`analytics`/`dbt_wh_defer` is stored as +`4:main|4:prod|9:analytics|12:dbt_wh_defer`. A target name and a schema are both +the user's own strings, so `prod|analytics` + `scratch` and `prod` + +`analytics|scratch` would otherwise be one key, and a profile moving between them +would read as the same environment rather than as one nothing has published. What +a message names is spelled out instead, never the encoded key. + +The target is the EFFECTIVE one, not the descriptor's `profile.target`: a +descriptor naming none inherits the workspace warehouse's, or the default in the +project's own `profiles.yml`, so reading the descriptor's would file every +inherited target under one empty name — and a `target.name` macro decides where a +model is built. + +The last two are in the key because deferring is resolving a relation NAME. A +warehouse repointed at another database, or a `profile.schema` moved by a +redeploy, keeps the first two while putting every relation somewhere else, and a +manifest is a list of relation names — there is no other way to notice. Keyed on +the first two alone, such a move would hand the next deferring run the names of +relations that are no longer there. Keyed on all four, it reads as an +environment nothing has published yet, which is what it is. + +What the key deliberately does NOT carry is the resolved connection. That is the +`profile_digest` a retry is held to, and it moves when a password is rotated, +which moves no relation; a warehouse pointing somewhere else entirely is +decision 11's accepted limitation, spelled the same way here as everywhere else. + +Today one script has one environment, because a descriptor fixes both the +warehouse and the target and a run cannot override either. The key is what makes +the *later* item — fork and preview environments — an addition rather than a +migration, and what makes a profile move detectable now. + +### Which runs publish it + +A successful `build` that did not itself defer, and whose graph becomes what the +script owns (`GraphRefresh::publishes_ownership`) — the same condition as the +graph's and the same reason: an invocation that scoped its own model set — a +`vars` or `select` override, or a descriptor dynamic by construction — describes +where the CALLER put those relations, not where this project's models live. +Publishing it would point every later deferral at one caller's scratch schema. + +**A run that deferred never publishes, whatever narrowed it**, and that is a +separate condition rather than a consequence of the first. A deferring run built +some of the relations its manifest names and resolved the rest out of the state +it read, so recording that manifest would claim relations nothing built — and a +model renamed since would be recorded under a name only a full build creates, +breaking every later deferral until one repairs it. `publishes_ownership` cannot +see this: it reads the caller's overrides, and a descriptor that already narrows +`select` needs none. + +A `retry` publishes nothing. Its `run_results.json` names only the nodes it +redid, so the environment would come to claim a run of a handful of models. The +environment's state is therefore the last full successful build, exactly as dbt +Cloud's "last successful run" is, and a run recovered by a retry leaves it at +the previous one. + +The AUTOMATIC in-job node retry is the same artifact under a different name: a +build it recovers is a successful build, but the `run_results.json` on disk is +the retry's. Such a run publishes the manifest **without** results, rather than +with a set describing some other slice of the build — the manifest is a function +of the project rather than of what ran, so deferral is unaffected. A `result:` +selector is the one thing left with nothing to read, and it is refused by name +against such a publication rather than passed to dbt (see "Selectors that read +the state" below). + +Under `test_behavior: after_all` the stored `run_results.json` is the test +phase's, because that is what the second invocation leaves in the target +directory — the same artifact a local `dbt run && dbt test` leaves behind. + +**What that condition means for what the artifacts may carry**, and why this +table is keyed by environment where `dbt_run_state` is keyed by principal. dbt +records the invocation's flags into `run_results.json`, and Windmill resolves +`$var:` / `$res:` references before dbt sees them — which is exactly why the +retry state is per-principal, so one caller's resolved `select` and `vars` are +not restorable by the next. Here they cannot be one caller's: a publishing run +added nothing of its own, and a descriptor that interpolates a `{{ }}` +placeholder into `vars` never publishes at all, so what is recorded is the +descriptor's own arguments — the script's content, which anyone entitled to run +it may already read. Widen the publish condition and that stops being true. + +### Where the blob goes + +`run_results.json` is small; `manifest.json` is not, and grows with the project +(535 KB on a two-model fixture). Each takes the same two homes: inline in the row +under `DBT_STATE_INLINE_MAX_BYTES` (8 MiB), and the INSTANCE's object storage +above it, with the row keeping the key. Inline is what makes the feature work on +an instance that has configured no storage at all; the ceiling is what stops one +project's manifest from becoming a multi-megabyte row rewritten by every run. A +project past the ceiling with no storage configured is told so, in the job log, +naming the setting and the variable — the run itself still succeeds, since +losing the state costs the next deferral rather than the build that just ran. + +**The instance store, not the workspace's**, which is where every other internal +worker artifact already lives (bun bundles, python wheels, job logs, the global +cache). The workspace bucket is the one members read and write through +`job_helpers/*` and `wmill.write_s3_file` with a caller-supplied key, and only +`volumes/` is reserved there — so a manifest under it is one any member could +replace, and the next deferring run would hand dbt an attacker-chosen +`defer_relation` for every unbuilt `ref()` while holding the script's warehouse +credentials. Its compiled SQL would be readable there too, for a project the +reader may have no access to. The consequence to know: a project past the ceiling +needs the instance store configured, which is an EE feature, so on CE the ceiling +is the limit and `DBT_STATE_INLINE_MAX_BYTES` is how it moves. + +Each publication writes its OWN keys +(`wmill_dbt_state///./`) +and the row switches to them in one statement, so an upload never overwrites an +artifact the committed row still names: a run that fails between its two uploads, +or between them and its row, leaves the state pointing at the pair it already +had. The objects the commit displaced are dropped afterwards, never before, since +a reader that has already read the row is about to fetch them; a reader that +loses that race re-reads for as long as the row keeps MOVING, rather than +reporting a state that is there. A reader takes no lock, so successive +publications can each overtake one; an unmoved row whose objects are gone is the +error that means what it says, and a bound on the re-reads is the other, for a +project republishing faster than a run can read. What a publication uploaded and then could not commit is dropped on the +way out — except after a commit that REPORTED an error, where what was lost may +be only the acknowledgement: dropping then would leave a committed row naming +objects that are gone, so an orphan is the cheaper side to take. + +The path and the environment are only a prefix of that key. The row is what says +where an artifact is, which is why state can travel with a renamed script and go +on naming objects under the old path's digest. The rest of the key is the job and +a per-EXECUTION nonce — zombie recovery re-runs a job under its own id, so keyed +on that alone a second attempt would overwrite the objects the first attempt's +committed row still names, then read those keys back as displaced and drop them. + +Publishers of one environment serialize on `pg_advisory_xact_lock`, so only one +of them settles the row and the objects it displaces at a time — an advisory lock +rather than the row's, because the first publish of an environment has no row to +lock and is exactly when two runs of a newly deployed script are most likely to +race. + +### Retention + +None, deliberately, and this is where it differs from the graph tables next +door. Those are pruned by age by the dbt runs themselves because their reader is +a transient run page. This one holds a single row per script per environment, +replaced in place, so it does not grow with runs — and its reader is every later +run of that script, so a project that runs monthly must still find last month's +state. It goes with the script instead: a path no live dbt version occupies any +more clears it, alongside `dbt_run_state` (`clear_dbt_script_state`, +`clear_dbt_script_state_if_path_retired`). + +The write carries a guard of its own, and it names the VERSION rather than the +path: the live dbt script there must be the one this job ran, or a later version +of it (`hash = $n OR $n = ANY(parent_hashes)`). "Some live dbt script is here" — +which is what the retry state settles for — is also satisfied by a script created +at a path this one was renamed away from, and this job's manifest would then +become that project's deferral state. A preview names no version and so publishes +nothing, which is right for a run of content that was never deployed. + +The job's KIND is checked beside it, because a preview carries a caller-supplied +`script_hash` into `runnable_id` (`run_preview_script`): the version alone would +let anyone who may run a job publish arbitrary content as a deployed script's +state. A flow or app step naming a deployed dbt script by path is an ordinary +`script` job carrying that script's own hash, so it publishes like any other run; +only INLINE flow code is a `FlowScript`, and that has no deployed version to +publish for. + +That guard HOLDS the script row (`FOR SHARE`) for the rest of the publication, so +a rename, archive or delete of the path either waits for it or is seen by it. +Read unlocked, it leaves a window where the lifecycle clear finds no row to take, +finishes, and the publication then commits state at a path a new script goes on +to occupy. The script row is taken before the sidecar, which is the order every +other dbt writer takes and what keeps the two off a deadlock. + +An artifact too large for its row is left in the store when the row is cleared, +as a deleted script leaves its bundle: reaching it from the delete would mean an +object-store client in `windmill-common` and a delete that has to land after the +caller's transaction commits, for one object per environment of a script that is +gone. + +### Asking for it + +`defer` is a field on the `build` command block, defaulting to the descriptor's +own `defer:`. A per-run toggle rather than a descriptor-only setting, because the +run that publishes an environment's state and the run that defers to it are two +invocations of ONE script (decision 6: N scripts means N projects): a project +that could only defer by descriptor could never populate the state it reads. + +A project whose profile selects its schema or database with a TEMPLATE — either +delimiter, since dbt renders `{% … %}` blocks as well as `{{ … }}` — is refused a +deferral outright, and publishes no state either: dbt renders those and Windmill +does not, so two renderings resolve to one `relation_root`, and a +deferral after the value changed would resolve every unbuilt `ref()` through the +previous location's manifest. Both sides, because a published template would sit +under a key a literal profile shares, and de-templating later would make that +stale manifest readable as the new location's. It covers a project-owned +`profiles.yml`, a `dbt_profile` resource — one block of the user's own file, +copied through unchanged — and a `profile.schema` written as given. Plainly +absent is different: that is the adapter's default, which does not move. + +A run that asks to defer with nothing published is refused, naming the +environment and the runs that cannot publish one. The alternative — running +without deferral — fails deep inside dbt with a relation-not-found the caller has +no way to connect back to a missing state. An agent worker is refused the same +way and for a reason it can act on: it reaches the database only through the API, +which does not expose this table. + +A `show` defers too, and every engine takes the flags on it. It compiles the +model it previews, so a model whose upstream this environment built and this run +did not is exactly the case a deferral exists for. So does the `dbt ls` that +resolves what a run's selection owns, without which a `result:` selector — which +reads `run_results.json` out of the state directory, and which `select` passes to +dbt verbatim — would fail before the build that would have honoured it. + +The result carries `deferred_to`, the run whose state was used. Without it what +a deferring run built against is unrecoverable, since the next successful run of +that environment replaces the state. + +### Selectors that read the state, and why they are refused rather than passed + +`--state` also feeds dbt's own selector methods, so publishing the state is what +makes `state:modified+`, `state:new` and `result:error+` resolve at all. Only a +deferring run is handed the directory, so a `state:` or `result:` method in +`select` or `exclude` without `defer` is refused before dbt starts. + +Refused, rather than left to dbt, because the engines disagree about it and two +of the three disagree silently. Given a state selector and no `--state`, +dbt-core 1.x raises (`Got a state selector method, but no comparison manifest`, +exit 2), but dbt-sa-cli 2.x and fusion read a MISSING state as an EMPTY one and +exit 0: `state:modified` then selects nothing and the run reports success having +built nothing, while `state:new` selects everything, because against an empty +state every node is new. A scheduled run that quietly stops doing work, or +quietly rebuilds the project, is the failure this state exists to prevent. + +From the DESCRIPTOR they are refused whether or not the run defers, and the +message says so. That selection is also what decides which nodes the script owns, +and the deploy resolves it before any run exists, with no state to compare +against. "Whatever changed last" is not an ownership answer. They describe one +run, so they belong in a run's own `select`. + +`source_status:` is refused under any setting: it compares `sources.json`, which +`dbt source freshness` writes and no run publishes here, so there is nothing to +compare against even while deferring. + +Two more refusals follow from the same argument, that a selector with nothing to +read must say so rather than resolve to a silent answer: + +- A `result:` method while deferring to a state that carries **no** + `run_results.json`. Publishing that is deliberate — a build recovered by + automatic node retry stores the manifest alone, its results describing the + retried nodes rather than the build ("Which runs publish it") — so `defer` + being on is not enough to know the file is there. Answerable only once the + state is loaded, so it is checked right after, naming the run that published. +- Any of them on a `parse`. A parse resolves a selection to store the graph and + never defers, so `defer` would not hand it a state at any setting, and the + remedy the other refusal offers would lead nowhere. It says that instead. + +Matching nothing is then an ordinary outcome for these methods, and for no +others. `state:modified+` selects the empty set exactly when nothing changed +since the published state, which is the answer a CI run wants, so a selection +naming a `state:` or `result:` method may resolve to no nodes. Such a run scoped +its own selection, so what it stores is a snapshot of its own and never what the +script owns, and nothing is un-wired by the empty set. + +The exemption is by METHOD, not by who chose the selection. Exempting every +caller-chosen one would take a misspelled model name, which resolves to nothing +just as surely, and report it as a build that did its work. An ordinary selection +matching nothing stays refused, from a run as from the descriptor — from the +descriptor because that one also decides ownership. + +Only what `select` and `exclude` spell directly. A method reached through a +`selectors.yml` definition is named nowhere the worker reads, and dbt's own +behaviour — including the silent one — is what stands there. + +### `--state` is also a retry's own argument, and that is a trap + +`dbt retry` reads the run it RESUMES from `--state`. Handed the deferral's +directory it resumes the successful run stored there, finds nothing failed, and +reports a green retry having rebuilt nothing — silently, on dbt-core 1.x, which +warns and exits 0. + +dbt-core 1.x has `--defer-state`, the deferral-only half of the pair, so a retry +there passes that and leaves `--state` alone. The Rust engines do not have it, +and a run that deferred is refused a retry on them, before the build: the +alternative is rebuilding the failed nodes with every `ref()` resolving into the +schema this run writes into, which for the narrowed run a deferral exists to +serve means writing them somewhere they do not belong. The automatic in-job node +retry is dropped for the same reason and says so in the log. + +The state directory is passed RELATIVE (`wm_dbt_state`, beside `wm_target` in the +job directory). dbt records the invocation's flags into `run_results.json` and a +later `dbt retry` restores them, so an absolute path would name the job directory +of the run being resumed, which is gone by then. Relative, it resolves against +the project root — whichever job directory the retry landed in. + +Three engine facts found while wiring this up, all worth knowing before filing a +bug against the feature. `dbt retry` on dbt-core 2.x restores **neither** the +resumed invocation's `--vars` nor its deferral: it re-parses with the current +(empty) ones, so a retry of a run that overrode `vars` rebuilds into the +descriptor's schema rather than the run's. That is independent of deferral and +predates it; the refusal above stops the deferring case from being the way it is +discovered. `dbt show` on either Rust engine prints a bare JSON array where +dbt-core frames it as `{"node": …, "show": […]}`, which `run_show` is written +against — so a preview there fails to parse whether or not it defers, and the +deferral itself resolves correctly under it. And neither Rust engine reached +dbt's own service-backed State (`--manage-state`) on any run measured here, so no +flag is passed to disable it. + +Because `select` reaches dbt verbatim, a deferring run also has a `--state` +directory for `result:` selectors, which is why `run_results.json` is stored +beside the manifest rather than the manifest alone. + ## Two decisions the implementation narrowed -**Decision 13 — no S3 copy of the manifest.** The sidecar holds every field the -graph renders; nothing reads a stored `manifest.json`, so writing one to S3 -would be an unread copy of data that is already reproducible by redeploying (or, -for a dynamic descriptor, by the next run). Worth adding the day something needs the -parts the sidecar drops — compiled SQL, macro definitions — and not before. +**Decision 13 — the manifest is stored once per environment, not per version.** +The sidecar holds every field the graph renders, so a copy of `manifest.json` +bought the graph nothing: it is reproducible by redeploying, or for a dynamic +descriptor by the next run. Deferral is the reader that changed that — it +resolves an unbuilt `ref()` through a manifest, and one on worker-local disk +answers for a machine's history rather than for the environment. So exactly one +manifest is kept per (script, environment), replaced by each successful run, +rather than one per version (see "Durable state per environment" above). -**Decision 14 — column lineage is not available.** The decision assumed -`manifest.json` carries column-to-column edges; it does not, in either core -engine. What it does carry is declared column *descriptions*, which are -ingested. Real column lineage would need Fusion (which does static analysis) or -a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. +**Decision 14 — column lineage comes from the parquet index, not the manifest.** +`manifest.json` carries no column-to-column edges, in any engine, and its +`columns` are the ones an author declared in `schema.yml`. Both halves exist in a +different artifact: `dbt compile --static-analysis strict --write-index` writes +`target/index/`, and two of its tables are `dbt.column_lineage.parquet` +(`from_node_unique_id`, `from_column_name`, `to_node_unique_id`, +`to_column_name`, `lineage_kind`) and `dbt.node_columns.parquet` (every column of +every node, with its declared type, its inferred type and its description). + +Three measured properties decide the shape of the ingest. + +**Strict analysis is a stricter dialect.** `select no_such_column from +ref(...)` is `UnresolvedIdentifier (dbt0227)` and exit 1 under `strict`, and +compiles fine under `baseline` (the default). So this is a separate `dbt compile` +with its own `--target-path`, never a flag on the build, and it is opt-in per +project: `column_lineage: true` in the descriptor. Off, nothing changes. On, a +project that cannot be analyzed keeps exactly the graph it had. + +The pass is best-effort about everything that is ITS: a wrong engine, a rejected +analysis, a missing or unreadable artifact, an over-long output and outrunning its +own time budget all degrade to partial lineage or none, plus a line in the job +log saying which. It is not best-effort about the JOB: a cancellation or the job's +own deadline fail it, because swallowing those would let a run that blew its +timeout inside an optional annotation publish a graph and report success. That +split is why the two halves have separate error contracts — the compile owns the +job's semantics and may `Err`; nothing the artifact does or fails to do is a +reason to fail a job, so an absent, unreadable or partial index is a value. The +decode still runs under the job poller, which both heartbeats through it and +ends it if the job is cancelled or completed meanwhile: the job reaching in, not +the artifact reaching out. The +budget is half the job's remaining wall clock, spent on the compile alone, so the +build that follows cannot be starved by it. + +**A failed pass still writes the index**, holding every edge of the models that +did analyze, so the artifact is read whatever the exit status and partial lineage +is a normal outcome. An unreachable *source* is milder still: `RemoteError +(dbt1014)` downgrades that model to `static_analysis: off` and the compile +succeeds. (Strict analysis queries the warehouse catalog for source schemas; a +`ref()`ed model is inferred statically and needs no built table.) + +**The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 — the version +`DBT_CORE_2X_VERSION` pins — accepts `--write-index` and `--write-lineage`, and +its own `views.sql` declares views over both tables, but it writes neither +parquet; only Fusion does today. The ADAPTER decides too: an experimental one +(postgres under `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`) turns static analysis off and +says so only in a warning on an otherwise successful compile. The gate is +therefore "the engine has the flag" (everything but 1.x, whose Python CLI has no +such option) plus "the file appeared", so a later release picking the feature up +needs no change here — and the job log carries the engine's own stderr whenever +no index appears, since without it "no column lineage" has no explanation. + +`lineage_kind` is stored as TEXT, not an enum. Three values exist — `copy` +(passthrough), `mod` (transformed) and `scan` (the column was read to produce the +ROW rather than the value: a join key, a `where` predicate, a `group by`) — and +the engine's own reader maps those three and passes anything else through, so the +set is the engine's to extend. All three are stored, and `copy`/`mod` are kept +first when the bound bites: a `scan` edge reaches every output column of its +model, so it is most of what a project's index holds and would draw as a complete +bipartite graph. Keeping it in the table anyway is what lets a later "show +indirect" view ask for it without every project being redeployed. + +Storage mirrors `dbt_edge` exactly: `dbt_column_edge`, keyed by (path, version, +job) with the same composite foreign key to `script`, so a version's column +lineage dies with the version and a run's snapshot with the sweep. + +**A table of its own, not `dbt_edge.column_lineage` JSONB.** Hanging the links on +the `ref()` edge they sit beneath would inherit its clone, prune, clear and +cascade paths for free, and it does not work: a model reading `{{ this }}` gets +column lineage from itself to itself, and `parent_map` has no self-loop, because +a model does not `ref()` itself. Those pairs have no `dbt_edge` row to attach to. +The loss is not hypothetical — an incremental that selects from `{{ this }}` +(`coalesce(p.dbl, s.dbl)`, `p.up as prev_up`) yields `up → prev_up` with kind +`copy`, a drawn edge meaning "this column carries the previous run's value". +Inventing self-loop `dbt_edge` rows to hold it is not an option either: that +table is `ref()` lineage. The typed column list lands in +`dbt_node.column_schema`, beside `columns` rather than merged into it — +`columns` stays what the author *declared*. + +Two things are user-visible. `column_schema` — every column of a relation, typed +and in the order the model emits them — rides the asset graph the details pane +already fetches, and replaces a panel that could only list the columns an author +happened to document. The edges are served by an endpoint of their own, +`assets/column_lineage`, which the pane asks for the selection it is drawing. + +Both are gated on being able to read the producing project, like the model's SQL: +a column-level view is the shape of what the author wrote, one level finer than +the `ref()` graph, which is ungated only because it draws relations the caller +already sees. A share-link viewer entitled to a dbt run therefore gets its +relations and `ref()` edges, and neither the SQL nor the columns. + +**One request per selection, and the gate re-decided per project.** The endpoint +takes every relation the view has reached and answers their union, because one +selection reaches several — a script's output column can derive from columns of +several models. Holding partial answers between selections instead was tried and +is what a client cache is: it produced a wrong premise for a relation two projects +describe, then staleness on redeploy, then a lost retry. + +The answer is the connected component around those relations, and that component +does not stop at the project that owns them: a relation one project produces is +another's source, so the walk resolves owners, reads their edges, walks, and +repeats for the relations that walk newly reached. Resolving once — for the +relations asked about — stops the trace at the first project boundary. The +security half is that a project reached this way is a project the caller may not +be entitled to, so the scope filter and the project's visibility are re-applied to +every project the expansion discovers, not decided once for the first owner set. + +A PINNED answer is the exception and needs none of it, whether it names a job or +a deployed version: the pin says which stored graph is on screen, and another +project's live graph is not part of it, so it answers for that one project. The +dbt editor pins by version on every selection and the run page pins by job; the +pipeline page pins nothing, and is where the walk crosses projects. + +**The component is bounded, and says when it was cut.** The renderer draws a box +per column, so a component past a few thousand edges is unreadable however it is +served — a synthetic 3000-model project whose models share a column returns 58k +direct edges and 7.3MB. The walk is breadth-first from the asked-for relations and +stops at 5000 edges, so what survives is the part nearest the selection rather +than an arbitrary slice, and `truncated` says so: a trace that stops short is +otherwise indistinguishable from one that ends. The FETCH is not bounded the same +way — a project's edges arrive whole, because the walk is what decides which of +them are in the component, and a `LIMIT` would cut a set that need not contain +the asked-for relation at all. What bounds it is the ingest's own cap per version +plus a stop on discovering further projects once the held set is outsized. + +The walk is in Rust rather than a recursive CTE. `EXPLAIN ANALYZE` on that same +project measured 1243ms against 59ms for the query alone: a CTE has no index to +walk, so the recursive term rescans the doubled edge set once per level (11.7M +rows), while the same walk over a map is microseconds. + +The two halves of a trace — dbt's and the pipeline's — meet at shared node ids. A +DuckDB script's `// column x <- dbt://wh/s/model.col` mints the same +`(dbt, path, column)` node dbt's own lineage does, so the producer graph the asset +graph already carries and the dbt graph are MERGED rather than chosen between, and +a trace crosses that boundary in either direction. What the browser cannot close +in one request is a relation the server discovers whose columns are consumed by a +script that writes into a third project: the producer half of that hop is the +canvas's, not the server's, and the seeds were computed before the answer arrived. + +**The analysis pass takes the build's own `--full-refresh`.** `is_incremental()` +branches on it, so an incremental model reading `{{ this }}` compiles its +self-join — and any `ref()` inside that branch — only when the flag is absent. A +pass that used the descriptor's default while the run overrode it would store +lineage for SQL that run never executed. For the same reason an invocation that +overrides the flag counts as `per_run_models`: its graph is its own, keyed to the +job, rather than standing as the version's. + +That flag is not the whole of it, and the rest is a property rather than a bug to +fix. `is_incremental()` is also false when the target table does not exist, so an +incremental model has **two shapes and one ingest holds one of them**: a deploy +before the first build compiles the cold shape, and the same project deployed +again once its tables exist compiles the incremental one. A static descriptor +re-ingests on neither runs nor time, so what is stored stays whatever the compile +in front of it saw. dbt has no mode that emits both, and re-analyzing per run +would buy a second `dbt compile` on every build to keep a graph nobody asked to +refresh. The contract is therefore the honest one: a version's graph describes +the compile that produced it, and a run that re-ingests describes its own run. ## Concept mapping @@ -1167,7 +1738,9 @@ a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. | `materialized: incremental` | `append` or `merge` (by `unique_key`) | same | | `{% snapshot %}` | `scd2` | same, incl. `_current` handling | | `unique`/`not_null`/`accepted_values`/`relationships` | `data_tests` | exact 1:1 with the four `// data_test` kinds | -| declared column metadata | `columns` on the asset node | descriptions only; see the note below | +| declared column metadata | `columns` on the asset node | descriptions only, from the manifest | +| analyzed column schema | `column_schema` on the asset node | `dbt.node_columns.parquet`, opt-in | +| column-to-column lineage | `dbt_column_edge` rows, drawn as a column trace | `dbt.column_lineage.parquet`, opt-in | | model `tags` | node badge | `tag` | | source freshness | `freshness` | `last_success_at` chip | | `run_results.json` | materialization records | `record_materialization` | @@ -1195,7 +1768,10 @@ render through the existing `RunnableNode.svelte` / `AssetNode.svelte` / on the canvas mid-run. `record_materialization` per model. Profile and select pickers in the editor. Per-model failure triage in the run view. -**Phase 4 (not in this PR).** `--defer` and `state:modified`. Partition and +**Phase 4 (not in this PR).** Slim CI: the fork and preview environments a +deferral would name instead of its own. The selectors themselves are here, since +`state:` and `result:` read the published state like any deferral does; what is +missing is a per-branch environment to compare a CI run against. Partition and backfill integration so `BackfillRangeDialog.svelte` works on dbt models. `wmill dbt import ` reading `DbtDag(...)` kwargs. @@ -1215,18 +1791,33 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: script reading one of the marts gets an edge to it. 6. **Shared node**: a native script that READS a mart renders as a reader of the same node the dbt model writes — one node, not two islands. Declared with a - plain read (`# dbt://`), never `# on`: a `dbt://` subscription is - refused at deploy, because nothing but dbt writes a warehouse relation and a - dbt run does not dispatch (see "no cascade from dbt"). -7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each + plain read (`# dbt://`), never `# on`: a subscription to a relation dbt + alone builds is refused at deploy, since a dbt run does not dispatch. +7. **Declared write**: a native `// materialize manual dbt://` script + and a dbt project reading that relation as a `source` render as one node; a + run of the script records its materialization and wakes a + `# on dbt://` subscriber — a subscription only that producer makes + wakeable, the dbt project reading the relation being no producer of it (see + "no cascade *from* dbt"). +8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each build only the expected subset. -8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph +9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph from the run's own manifest, so a model that placeholder enables appears in the same run that builds it. -9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's +10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's own `profiles.yml` with env-var injection. -10. **Caching**: a second run reuses the cached `dbt_packages/` with no network +11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. +12. **Deferral**: a full run publishes the environment's state; a second run + that builds one downstream model into another schema resolves its unbuilt + `ref()` to the relation the state names, where the same run without `defer` + fails with relation-not-found. +13. **State selectors**: with a state published, `state:modified+` selects + nothing while the project is unchanged and exactly the changed model and its + children after one is edited. Without `defer` it is refused rather than + passed, and a `result:` selector against a state published by a + node-retry-recovered build is refused too, that one carrying no + `run_results.json`. Keep only tests that pin behavior a future change could break. Per AGENTS.md, delete development scaffolding before marking the PR ready. diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 32fff7ca23..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion (EL)"; this section records only what future feature work must not break. -- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and - `manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK +- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected + elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** — diff --git a/docs/reusable-ai-agents.md b/docs/reusable-ai-agents.md index 8383bc2836..ede8f64b1d 100644 --- a/docs/reusable-ai-agents.md +++ b/docs/reusable-ai-agents.md @@ -22,22 +22,54 @@ every workspace via the standard cached-resource-type sync, like other built-in or flow expressions), so saving round-trips losslessly. Each host flow overrides what it needs: `tool_inputs` stores per-tool overrides (a diff from the resource tool's own transforms) that overlay onto the matching tools at runtime. Editing on a linked step edits - the flow's use of the agent; editing under the "Editing" banner edits the agent itself. + the flow's use of the agent; editing in the agent editor edits the agent itself. In the flow editor, the AI agent step's **Step Input** tab shows a single read-only card (*linked to *, with the inherited brain + tools and an explanatory tooltip) plus -*Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink* -(fork the resolved config — including any `tool_inputs` — back into the step as a one-off). -While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking -first when there is something to drop), and the unsaved-changes badge opens a diff against the -deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of, -and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so -a reload brings the step back as a standalone agent with no path to save back to. +*Edit*, which opens the agent editor over the flow, and *Unlink* (fork the resolved config — +including any `tool_inputs` — back into the step as a one-off). A linked agent's tools appear as display-only graph tool nodes (clicking one selects the agent step); below the step's inputs, each tool gets a section with the standard schema-aware input editors (prop picker included) and a read-only view of its code — edits persist into `tool_inputs`. +## Drafts + +The agent editor edits the resource through a **per-user resource draft** (`draft` table, +`item_kind = 'resource'`), autosaved by `useAgentDraft` and deployed by the editor's own Deploy +button. It is the same draft row the generic resource editor writes and the Review & Deploy page +lists, so an agent can be deployed from any of them. + +A flow does not wait for that deploy to see the draft: + +- Testing the flow, or a single linked step, runs the draft. `runFlowPreview` and `ModuleTest` + substitute each linked step for the standalone step the draft would run as + (`linkedAgentDrafts.ts`): `agent` cleared, the draft's brain as static input transforms, the + draft's tools on the step, and the step's own `user_message`/`user_attachments` kept on top — + the same overlay order `ai_executor.rs` applies to a linked step. `tool_inputs` is untouched, + since the worker overlays it in both branches. +- The step's linked card and the graph's tool nodes show the draft, with a *Draft* badge, so the + editor describes what a test would run. Read-only surfaces (the deployed flow page, the run + viewer) stay on the deployed agent: they resolve tools through `publishLinkedAgentTools` without + the draft flag. +- Deploying the flow lists every linked agent that has a draft in the confirmation dialog, beside + the draft triggers. Deploying one writes the resource and drops the draft; leaving one out keeps + its draft untouched, and the flow runs the agent as currently deployed. That is the one place + the two kinds differ: an undeployed draft trigger is deleted, because it belongs to the flow, + while an agent draft belongs to a resource other flows also use. + +Because a draft is per-user, a flow test can behave differently for two people looking at the same +flow. That is the same contract as a flow draft, and deploying the agent is what makes it shared. + +Inlining has a consequence worth knowing: a preview job's `raw_flow` then carries the agent's +config, where a linked step used to carry only the path and leave the resolution to the worker. So +an agent's prompt and tool set are readable by whoever can read that preview job, which is a wider +set than whoever can read the resource when the agent sits in a more restricted folder than the +flow. No credential travels with it — the provider stays a `$res:` reference, resolved at run time +as the runner. The agent editor's own test pane has inlined the same way since drafts existed; +closing the gap would mean the preview carrying a draft *reference* the worker resolves, rather +than the config. + Sharing works through standard resource folder permissions (save agents under `f/...`). Only the agent's brain is interpolated when the step runs. A tool's own `$res:`/`$var:` defaults are diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3fb5b494c2..4761b01b04 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index d4dedb34e9..f8d4261f93 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/attachments/newTabModifier.dom.test.ts b/frontend/src/lib/attachments/newTabModifier.dom.test.ts new file mode 100644 index 0000000000..115877298c --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.dom.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { newTabModifier } from './newTabModifier.svelte' + +const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent }) +const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)' +const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + +const attached: (() => void)[] = [] + +/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */ +function pill() { + const node = document.createElement('span') + document.body.append(node) + const modifier = newTabModifier() + const cleanup = modifier.attach(node) as () => void + attached.push(cleanup) + const hover = (init: MouseEventInit = {}) => + node.dispatchEvent(new MouseEvent('mouseenter', init)) + const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init)) + const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave')) + return { modifier, hover, move, unhover, cleanup } +} + +const keydown = (init: KeyboardEventInit) => + window.dispatchEvent(new KeyboardEvent('keydown', init)) + +describe('newTabModifier', () => { + // The window listeners outlive the DOM, so every case has to be torn down through the + // attachment rather than by emptying the body. + afterEach(() => { + attached.splice(0).forEach((cleanup) => cleanup()) + document.body.replaceChildren() + vi.unstubAllGlobals() + }) + + // The hover event carries the live modifier state, so a modifier pressed before the pointer + // arrived (or while this window was unfocused) is picked up rather than read as false. + it('seeds from the hover event, per platform', () => { + onPlatform(LINUX) + const linux = pill() + linux.hover({ ctrlKey: true }) + expect(linux.modifier.held).toBe(true) + + onPlatform(MAC) + const mac = pill() + // macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier. + mac.hover({ ctrlKey: true }) + expect(mac.modifier.held).toBe(false) + mac.hover({ metaKey: true }) + expect(mac.modifier.held).toBe(true) + }) + + // Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase + // listener would go blind whenever focus sits in one. + it('sees a keydown that a focused element stops from propagating', () => { + onPlatform(LINUX) + const { modifier, hover } = pill() + hover() + const input = document.createElement('input') + input.addEventListener('keydown', (e) => e.stopPropagation()) + document.body.append(input) + + input.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true }) + ) + expect(modifier.held).toBe(true) + }) + + // A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown + // on the way back, while the pointer parked on the pill fires no fresh mouseenter either. + it('re-seeds from pointer movement after the window lost focus', () => { + onPlatform(LINUX) + const { modifier, hover, move } = pill() + hover({ ctrlKey: true }) + window.dispatchEvent(new Event('blur')) + expect(modifier.held).toBe(false) + + move({ ctrlKey: true }) + expect(modifier.held).toBe(true) + }) + + it('stops tracking once unhovered', () => { + onPlatform(LINUX) + const { modifier, hover, unhover } = pill() + hover({ ctrlKey: true }) + unhover() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) + + it('stops tracking when the element is destroyed while hovered', () => { + onPlatform(LINUX) + const { modifier, hover, cleanup } = pill() + hover({ ctrlKey: true }) + // Hovering again without leaving must not strand the first hover's listeners, which nothing + // would then hold a reference to. + hover({ ctrlKey: true }) + cleanup() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) +}) diff --git a/frontend/src/lib/attachments/newTabModifier.svelte.ts b/frontend/src/lib/attachments/newTabModifier.svelte.ts new file mode 100644 index 0000000000..06c69f36cc --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.svelte.ts @@ -0,0 +1,66 @@ +import type { Attachment } from 'svelte/attachments' +import { isMac } from '$lib/utils' + +/** + * Tracks whether the modifier that turns a click into a new browser tab is held, but only while + * the attached element is hovered, which is the only moment the answer is used. + */ +export function newTabModifier() { + let held = $state(false) + + // Only the modifier that actually yields a tab: shift opens a window, alt can start a + // download, and on macOS ctrl+click is a secondary click. + // Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to + // a focus change cannot strand the flag on. + const sync = (event: KeyboardEvent | MouseEvent) => { + held = isMac() ? event.metaKey : event.ctrlKey + } + const clear = () => { + held = false + } + + const attach: Attachment = (node) => { + // One controller per hover: a mirrored remove list leaks any listener whose options drift. + let hover: AbortController | undefined + const leave = () => { + hover?.abort() + hover = undefined + clear() + } + const enter = (event: MouseEvent) => { + // Seeded from the hover itself: mouse events carry the same modifier flags as key events, + // so a modifier already held before the pointer arrived reads correctly. + sync(event) + // Re-entering without an intervening leave would strand the previous controller: nothing + // else references it, so its listeners could never be removed. + hover?.abort() + hover = new AbortController() + const { signal } = hover + // Same reason the hover seeds: a modifier held across a keyboard app switch delivers no + // keydown on the way back, so the pointer is all that is left to re-read it from. + node.addEventListener('mousemove', sync, { signal }) + // Capture: editors and menus stopPropagation the keys they handle, hiding the modifier + // from a bubble-phase listener whenever focus sits in one. + window.addEventListener('keydown', sync, { capture: true, signal }) + window.addEventListener('keyup', sync, { capture: true, signal }) + // Not capture, unlike the two above: blur does not bubble but does reach the window while + // capturing, so it would fire for every element that loses focus. + window.addEventListener('blur', clear, { signal }) + } + + const life = new AbortController() + node.addEventListener('mouseenter', enter, { signal: life.signal }) + node.addEventListener('mouseleave', leave, { signal: life.signal }) + return () => { + life.abort() + leave() + } + } + + return { + get held() { + return held + }, + attach + } +} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 320b3e36dd..7f2b646f6b 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -41,6 +41,13 @@ import TextInput from './text_input/TextInput.svelte' import { sameTopDomainOrigin } from '$lib/cookies' import SyncResourceTypes from './SyncResourceTypes.svelte' + import { + alphabetical, + byPopularity, + hubResourceTypePicks, + localResourceTypeCounts, + recordHubResourceTypePick + } from './pickerPopularity' import Label from './Label.svelte' import ResourcePathHint from './ResourcePathHint.svelte' @@ -332,6 +339,7 @@ export async function open(rt?: string) { if (!rt) { loadResourceTypes() + loadPopularity() } step = 1 //express && !manual ? 3 : 1 // The list is keyboard-driven from the search field, so it takes focus on open. @@ -378,12 +386,29 @@ } } + /** + * Orders the browse list: the types this workspace already has resources of lead, ranked + * among themselves by the hub's pick counts, then everything else on the same counts. + * `byPopularity` carries the full rule. Both signals are fetched, so the rows render + * alphabetically and re-sort when this lands. + */ + let popularity: (a: string, b: string) => number = $state(alphabetical) + + async function loadPopularity() { + if (!effectiveWorkspace) return + const [hub, local] = await Promise.all([ + hubResourceTypePicks(effectiveWorkspace), + localResourceTypeCounts(effectiveWorkspace) + ]) + popularity = byPopularity(hub, local) + } + async function loadConnects() { if (!connects) { try { - const list = (await OauthService.listOauthConnects()) - .filter((x) => x.name != 'supabase_wizard') - .sort((a, b) => a.name.localeCompare(b.name)) + const list = (await OauthService.listOauthConnects()).filter( + (x) => x.name != 'supabase_wizard' + ) connects = list.map((x) => x.name) connectsInfo = Object.fromEntries(list.map((x) => [x.name, x])) } catch (e) { @@ -466,19 +491,17 @@ // providers — so any of them can also be connected with the user's own // credentials or manually, not only via the shared instance setup (same as // the authorization-code behavior). - connectsManual = availableRts - .map( - (x) => - ({ - key: x, - ...(apiTokenApps[x] ?? { - instructions: '', - img: undefined, - linkedSecret: undefined - }) - }) as { key: string; img?: string; instructions: string[] } - ) - .sort((a, b) => a.key.localeCompare(b.key)) + connectsManual = availableRts.map( + (x) => + ({ + key: x, + ...(apiTokenApps[x] ?? { + instructions: '', + img: undefined, + linkedSecret: undefined + }) + }) as { key: string; img?: string; instructions: string[] } + ) const filteredNativeLanguages = filteredConnectsManual?.filter( (o) => nativeLanguagesCategory?.includes(o[0]) ?? false ) @@ -952,6 +975,10 @@ } }) } + // Saving is what "picking a type" means to the hub: reaching step 2 is still + // browsing. Both branches above count, `filling` included — an imported stub is + // a type taken into the workspace just the same. + recordHubResourceTypePick(effectiveWorkspace, resourceType) dispatch('refresh', path) dispatch('close') sendUserToast( @@ -972,6 +999,9 @@ if (step == 1) { loadConnects() loadResourceTypes() + // Opened on a specific type, `open()` skipped this; backing out to the browse + // list is the first time it is needed. + loadPopularity() } } @@ -980,24 +1010,35 @@ let filteredConnects: { key: string }[] = $state([]) let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([]) - // uFuzzy scores the name and the description as one string, so searching "google" ranks - // every type whose description mentions Google alongside the ones named after it. Re-sort - // on which field matched, keeping uFuzzy's order within a tier. + let searching = $derived(filter.trim() !== '') + + // Searching, the query owns the order: uFuzzy scores the name and the description as one + // string, so "google" ranks every type whose description mentions Google alongside the + // ones named after it — re-sort on which field matched, keeping uFuzzy's order within a + // tier. Browsing, there is no query to rank against, so popularity orders the list. const rank = (items: { key: string }[] | undefined) => items && - sortResourceTypesByMatch( - items, - filter, - (x) => x.key, - (x) => resourceTypeDescriptions[x.key] - ) + (searching + ? sortResourceTypesByMatch( + items, + filter, + (x) => x.key, + (x) => resourceTypeDescriptions[x.key] + ) + : // Both signals are keyed by resource type, and a sandbox client is a second row + // against one (`salesforce_sandbox` saves a `salesforce`), so it ranks on the + // parent's popularity. Its own key still breaks the tie the pair then have, or + // the two would order arbitrarily. + [...items].sort( + (a, b) => + popularity(stripSandboxSuffix(a.key), stripSandboxSuffix(b.key)) || + a.key.localeCompare(b.key) + )) let rankedConnects = $derived(rank(filteredConnects)) let rankedConnectsManual = $derived( rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined ) - let searching = $derived(filter.trim() !== '') - // Browsing, the "Others" list leads with the native database types. Searching, that // grouping would outrank the search itself — `ms_sql_server` sorting under `mysql` on // "sql" — so the ranked order stands on its own. diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 35bd88abec..7520cc5013 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -12,8 +12,8 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import { ClearableInput, Drawer, DrawerContent } from './common' import { sendUserToast } from '$lib/toast' - import { type ColumnDef } from './apps/components/display/dbtable/utils' - import DBTable from './DBTable.svelte' + import { renderDbEqualityFilter, type ColumnDef } from './apps/components/display/dbtable/utils' + import DBTable, { type DbForeignKeyTarget, type DbRowFilter } from './DBTable.svelte' import type { IDbSchemaOps, IDbTableOps } from './dbOps' import DropdownV2 from './DropdownV2.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -46,7 +46,12 @@ dbSupportsSchemas: boolean databaseIsEmpty?: boolean colDefs: Record | undefined - dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps + dbTableOpsFactory: (params: { + colDefs: ColumnDef[] + tableKey: string + /** Raw SQL predicate AND-ed into the reads (already escaped). */ + whereClause?: string + }) => IDbTableOps dbSchemaOps: IDbSchemaOps refresh?: () => void initialSchemaKey?: string @@ -214,6 +219,83 @@ : selected.tableKey ) + // Set by "Go to row" on a foreign-keyed cell; pinned to the table it was + // created for so a schema change can't carry it onto an unrelated table. + let rowFilter: (DbRowFilter & { tableKey: string }) | undefined = $state() + let activeRowFilter = $derived(rowFilter?.tableKey === tableKey ? rowFilter : undefined) + let whereClause = $derived( + activeRowFilter + ? renderDbEqualityFilter(activeRowFilter.column, activeRowFilter.value, dbType) + : undefined + ) + + function selectTable(schemaKey: string | undefined, table: string) { + rowFilter = undefined + selected = { schemaKey, tableKey: table } + } + + /** Where a foreign key's `schema.table` target lives in the sidebar, or + * undefined when it cannot be opened from here. */ + function resolveForeignKeyTarget( + targetTable: string + ): { schemaKey: string; table: string } | undefined { + const parts = targetTable.split('.') + const table = parts[parts.length - 1] + const qualifier = parts.length > 1 ? parts.slice(0, -1).join('.') : undefined + // Without schema support the sidebar browses the connection's default + // schema only, and unqualified reads would hit a same-named local table. + if (!dbSupportsSchemas && qualifier && qualifier !== selected.schemaKey) return undefined + const schemaKey = dbSupportsSchemas && qualifier ? qualifier : selected.schemaKey + if (!schemaKey || !(table in (dbSchema.schema[schemaKey] ?? {}))) return undefined + return { schemaKey, table } + } + + function goToRow(target: DbForeignKeyTarget) { + const resolved = resolveForeignKeyTarget(target.table) + if (!resolved) { + sendUserToast(`Table ${target.table} cannot be opened from this schema`, true) + return + } + if (renderDbEqualityFilter(target.column, target.value, dbType) === undefined) { + sendUserToast('This value cannot be used as a filter', true) + return + } + const { schemaKey, table } = resolved + selectTable(schemaKey, table) + rowFilter = { + tableKey: dbSupportsSchemas ? `${schemaKey}.${table}` : table, + column: target.column, + value: target.value + } + } + + // The result carries the table it was fetched for: `resource` keeps the + // previous value while refetching, and a stale list would decorate the new + // table's same-named columns as foreign keys. + let foreignKeys = resource( + [() => selected.tableKey, () => selected.schemaKey, () => colDefs], + async ([table, schema], _prev, { signal }) => { + if (!table) return undefined + const forTableKey = dbSupportsSchemas && schema ? `${schema}.${table}` : table + const fks = + features?.foreignKeys === false + ? [] + : await dbSchemaOps.onFetchForeignKeys({ table, schema }) + // A newer selection started meanwhile: an AbortError keeps this result + // out of `current`, where it would shadow the newer table's keys. + if (signal.aborted) throw new DOMException('Superseded', 'AbortError') + return { tableKey: forTableKey, foreignKeys: fks } + } + ) + // Only keys whose target the sidebar can open get the "Go to row" affordance. + let currentForeignKeys = $derived.by(() => { + const fetched = foreignKeys.current + if (!fetched || fetched.tableKey !== tableKey) return undefined + return fetched.foreignKeys.filter( + (fk) => fk.targetTable && resolveForeignKeyTarget(fk.targetTable) !== undefined + ) + }) + let askingForConfirmation: | (ConfirmationModal['$$prop_def'] & { onConfirm: () => void }) | undefined = $state() @@ -395,14 +477,12 @@ role="button" tabindex="0" onclick={() => { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) }} onkeydown={(e) => { if (e.key === 'Enter' || e.key === ' ') { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) } }} @@ -468,7 +548,7 @@ + diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 51026f10fd..6c89f1a039 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -112,17 +112,37 @@ } /** - * Creates a URL-synced filter instance that automatically syncs with URL search parameters + * Creates a URL-synced filter instance that automatically syncs with URL search parameters. + * + * `initial` supplies defaults for keys the URL doesn't carry (a route segment, a persisted + * toggle); the returned `seed` re-applies them after a navigation has rewritten the query. */ export function useUrlSyncedFilterInstance( - schemaRec: T - ): { val: Partial> } { + schemaRec: T, + initial?: Partial> + ): { + val: Partial> + seed: (values: Partial>) => void + } { // Build the Zod schema from the filter schema const zodSchema = filterSchemaRecToZodSchema(schemaRec) // Create URL-synced search params const urlFilter = useSearchParams(zodSchema) as Record + // A default has to arrive as a URL param: the URL→instance effect below drops whatever the + // URL lacks, so a value written to the instance is undone on the next sync. Going through + // urlFilter rather than straight to history keeps the search-param cells in step, so it + // does not matter whether a popstate follows. + function seed(values: Partial>) { + const sp = new URLSearchParams(window.location.search) + for (const [key, value] of Object.entries(values) as [string, unknown][]) { + if (value === undefined || value === null || sp.has(key)) continue + urlFilter[key] = value instanceof Date ? value.toISOString() : value + } + } + if (initial) seed(initial) + // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) @@ -173,7 +193,15 @@ }) } - return filterInstance + return { + get val() { + return filterInstance.val + }, + set val(v: Partial>) { + filterInstance.val = v + }, + seed + } } function filterToText(filter: FilterInstance, schema: F): string { diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 3a1abfce91..2ea94f3606 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -81,7 +81,18 @@ import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' import type { Trigger } from './triggers/utils' import { deployTriggers, handleSelectTriggerFromKind } from './triggers/utils' - import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' + import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte' + import { + agentDraftCanWrite, + linkedAgentPaths, + loadLinkedAgentDrafts, + type LinkedAgentDraft + } from './flows/linkedAgentDrafts' + import { agentDraftDeployRefusal } from './flows/agentDraft.svelte' + import { markAgentWritten } from './flows/agentEditorStore.svelte' + import { logReusableAgentUsage } from './flows/agentTelemetry' + import { deployDraft } from '$lib/utils_draft_deploy' + import { getUserExt } from '$lib/user' import { Triggers } from './triggers/triggers.svelte' import { StepsInputArgs } from './flows/stepsInputArgs.svelte' import { aiChatManager } from './copilot/chat/AIChatManager.svelte' @@ -175,8 +186,16 @@ let confirmCallback: () => void = $state(() => {}) // What happens when user clicks `override` in warning let open: boolean = $state(false) // Is confirmation modal open - // Draft triggers confirmation modal - let draftTriggersModalOpen = $state(false) + // Draft changes (triggers + linked agents) confirmation modal + let draftChangesModalOpen = $state(false) + /** The unsaved agent drafts the pending deploy found. Loaded rather than derived: it takes a + * request per linked agent, so it is resolved when the deploy asks. */ + let draftAgents = $state([]) + let agentCanWrite = $state>({}) + let agentRefusal = $state>({}) + + /** What the dialog's confirm hands back to `saveFlow`. */ + type DraftChangesToDeploy = { triggers: Trigger[]; agents: LinkedAgentDraft[] } // Top-bar responsive collapse. Measured via bind:clientWidth — we can't // rely on viewport `md:` because the editor lives inside other panes @@ -207,7 +226,7 @@ ] : [] ) - let confirmDeploymentCallback: (triggersToDeploy: Trigger[]) => void = () => {} + let confirmDeploymentCallback: (toDeploy: DraftChangesToDeploy) => void = () => {} // AI changes warning modal let aiChangesWarningOpen = $state(false) @@ -219,11 +238,57 @@ const job: Job | undefined = $derived(flowPreviewContent?.getJob()) let showJobStatus = $state(false) - async function handleDraftTriggersConfirmed(event: CustomEvent<{ selectedTriggers: Trigger[] }>) { - const { selectedTriggers } = event.detail + async function handleDraftChangesConfirmed( + event: CustomEvent<{ selectedTriggers: Trigger[]; selectedAgents: LinkedAgentDraft[] }> + ) { + const { selectedTriggers, selectedAgents } = event.detail // Continue with saving the flow - draftTriggersModalOpen = false - confirmDeploymentCallback(selectedTriggers) + draftChangesModalOpen = false + confirmDeploymentCallback({ triggers: selectedTriggers, agents: selectedAgents }) + } + + /** Deploy each selected agent's draft, the same way the Review & Deploy page deploys the same + * row: hand the path to `deployDraft` and let it promote whatever the draft holds, with no + * re-read to check it still matches what the dialog listed. Agents left out keep their draft + * untouched. */ + async function deployAgentDrafts(agents: LinkedAgentDraft[]) { + const ws = opWorkspace + if (!ws) return + for (const listed of draftAgents) { + // Only the rows the dialog gave a choice on. A `Read-only` or `Invalid config` agent can + // never be selected, so counting it as "kept" would record a decision the user was never + // offered and bias the pair towards keeping. + const selectable = agentCanWrite[listed.path] !== false && !agentRefusal[listed.path] + if (selectable && !agents.some((a) => a.path === listed.path)) { + logReusableAgentUsage('draft_kept_on_deploy') + } + } + for (const agent of agents) { + // Writes the resource, deletes the draft row, and clears the local hint and the workspace + // drafts cache. A failure aborts the flow save the way a failing trigger does, rather than + // deploying a flow against agents that were meant to change with it. + const deployed = await deployDraft('resource', agent.path, ws, { + draftOnly: agent.noDeployed + }) + if (!deployed.success) { + throw new Error(`Could not deploy agent ${agent.path}: ${deployed.error}`) + } + // Nothing was promoted: the draft had gone by the time the helper read it. Saying the agent + // deployed would be a lie about the one thing the toggle decides. + if (deployed.noop) { + throw new Error( + `The draft for ${agent.path} was deployed or discarded elsewhere while this deploy ran, so nothing was written for it.` + ) + } + // `deployDraft` deletes the server row but leaves any in-memory cell for this key, and that + // cell is what `agentDraftState` prefers — a still-mounted holder would otherwise keep + // feeding a phantom draft to the cards and to the next deploy dialog. Local only: `remove` + // would POST a second delete, debounced and past the baseline the first one cleared. + UserDraft.forgetLocal('resource', agent.path, { workspace: ws }) + // Every linked card and the graph key on this to refetch the agent they display. + markAgentWritten(ws, agent.path) + logReusableAgentUsage('draft_deployed_with_flow') + } } // Inside an AI session pane (SessionEditorTarget injects an aiChatManager via @@ -412,18 +477,53 @@ deployedBy = flow.edited_by } - async function saveFlow(deploymentMsg?: string, triggersToDeploy?: Trigger[]): Promise { - if (!triggersToDeploy) { - // Check if there are draft triggers that need confirmation + async function saveFlow(deploymentMsg?: string, toDeploy?: DraftChangesToDeploy): Promise { + if (!toDeploy) { + // Draft triggers and drafts on the agents this flow links: both are unsaved changes the + // deploy would otherwise leave behind, so they are confirmed together. const draftTriggers = triggersState.triggers.filter((trigger) => trigger.draftConfig) - if (draftTriggers.length > 0) { - draftTriggersModalOpen = true - confirmDeploymentCallback = async (triggersToDeploy: Trigger[]) => { - await saveFlow(deploymentMsg, triggersToDeploy) + try { + draftAgents = [ + ...( + await loadLinkedAgentDrafts(linkedAgentPaths(flowStore.val.value), opWorkspace) + ).values() + ] + } catch (err: any) { + // This runs before the try below, and `withAIChangesWarning` invokes its callback without + // awaiting, so a rejection here would be unhandled: the button would do nothing at all, + // with no toast and no `onDeployError`. Report it the way the rest of the save does. + // Deploying anyway is not the fallback — this throws only when an agent's unsaved changes + // cannot be read, which is exactly when the dialog must not claim there are none. + onDeployError?.({ error: err }) + sendUserToast(`The flow could not be saved: ${err?.body ?? err}`, true) + return + } + agentCanWrite = {} + agentRefusal = {} + if (draftAgents.length > 0) { + // One lookup for the whole list: an agent lives in a folder, and the groups and admin + // flag that answer for it are per workspace, so the nav user would answer for the wrong + // membership when a session editor operates on another workspace. + const user = await getUserExt(opWorkspace ?? '').catch(() => undefined) + agentCanWrite = Object.fromEntries( + draftAgents.map((a) => [a.path, agentDraftCanWrite(a, user ?? $userStore ?? undefined)]) + ) + // The path is passed, so a draft that renames the agent is refused here too: a rename is + // the resource editor's to deploy, and this dialog lists the agent under the path the + // flow links. + agentRefusal = Object.fromEntries( + draftAgents.map((a) => [a.path, agentDraftDeployRefusal(a.state, a.path)]) + ) + } + if (draftTriggers.length > 0 || draftAgents.length > 0) { + draftChangesModalOpen = true + confirmDeploymentCallback = async (confirmed: DraftChangesToDeploy) => { + await saveFlow(deploymentMsg, confirmed) } return } } + const triggersToDeploy = toDeploy?.triggers loadingSave = true try { @@ -454,6 +554,11 @@ // loadingSave = false // del // return + // Ahead of the flow itself, as the update branch deploys its triggers: an agent is a + // resource of its own, so the flow should land on top of the agent set it was tested + // against rather than the other way round. + await deployAgentDrafts(toDeploy?.agents ?? []) + // `newFlow` comes from the embedder, and updating a path that has no // deployed flow 404s. Confirm with the server before taking the update // branch so a first deploy still lands. @@ -631,7 +736,7 @@ const resolved = linkedAgentToolsForScope(to) for (const [moduleId, agentPath] of linkedAgentEntries(linkedAgentRefs)) { if (resolved[moduleId] === undefined) { - publishLinkedAgentTools(agentPath, ws, to, moduleId) + publishLinkedAgentTools(agentPath, ws, to, moduleId, true) } } } @@ -699,7 +804,7 @@ // writing overrides against — the tool ids of the agent that was just replaced. claimLinkedToolsFetch(scope, moduleId) clearLinkedAgentTools(scope, moduleId) - publishLinkedAgentTools(agentPath, ws, scope, moduleId) + publishLinkedAgentTools(agentPath, ws, scope, moduleId, true) } publishedAgentByModule = next }) @@ -1266,14 +1371,17 @@ currentValue={flowStore.val} /> - t.draftConfig)} + {draftAgents} + {agentCanWrite} + {agentRefusal} isFlow={true} on:canceled={() => { - draftTriggersModalOpen = false + draftChangesModalOpen = false }} - on:confirmed={handleDraftTriggersConfirmed} + on:confirmed={handleDraftChangesConfirmed} /> diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 47da29bb57..945533bcc8 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -67,6 +67,14 @@ const dispatch = createEventDispatcher() + // A bucket of this viewer's own, never the editor's. Both are keyed by (workspace, flow path), so + // a viewer mounted over the flow being edited — the version-history drawer, which renders the + // same path — would otherwise publish its deployed tools into the editor's bucket and replace the + // drafted tool nodes there. The editor's graph has to keep showing the tools a preview would + // actually run, and nothing republishes when the drawer closes. `FlowStatusViewerInner` scopes + // itself the same way, with `job:`. + let linkedToolsPath = $derived(`view:${flow?.path ?? ''}`) + // This read-only viewer doesn't run initFlowState, so linked agents' tools would otherwise never // resolve. Resolve them for display, keyed by module id. Best-effort: publishLinkedAgentTools // swallows access errors and publishes [], so an inaccessible agent simply shows no tool nodes @@ -80,7 +88,15 @@ for (const m of modules) { const value = m?.value as { type?: string; agent?: string } | undefined if (value?.type === 'aiagent' && value.agent) { - publishLinkedAgentTools(value.agent, ws, linkedToolsScope(ws, flow?.path), m.id) + // Without the draft: this viewer shows a deployed flow or a past run, both of which + // used the deployed agent. + publishLinkedAgentTools( + value.agent, + ws, + linkedToolsScope(ws, linkedToolsPath), + m.id, + false + ) } } }) @@ -101,6 +117,7 @@ earlyStop={flow?.value?.skip_expr !== undefined} cache={flow?.value?.cache_ttl !== undefined} path={flow?.path} + {linkedToolsPath} {download} minHeight={fillAvailableHeight ? Math.max(minHeight, availableHeight) : minHeight} {workspace} diff --git a/frontend/src/lib/components/FlowLoopIterationPreview.svelte b/frontend/src/lib/components/FlowLoopIterationPreview.svelte index 1378e20f04..0a4f5c60ee 100644 --- a/frontend/src/lib/components/FlowLoopIterationPreview.svelte +++ b/frontend/src/lib/components/FlowLoopIterationPreview.svelte @@ -5,6 +5,7 @@ import { createEventDispatcher, getContext } from 'svelte' import type { FlowEditorContext } from './flows/types' import { runFlowPreview } from './flows/utils.svelte' + import { sendUserToast } from '$lib/toast' import SchemaForm from './SchemaForm.svelte' import FlowStatusViewer from '../components/FlowStatusViewer.svelte' import FlowProgressBar from './flows/FlowProgressBar.svelte' @@ -101,15 +102,22 @@ // The preview flow holds only the loop body, so it inherits none of the flow's settings: // carry the tag over so the iteration lands on the worker group the flow runs on. const newFlow = { value: { modules }, summary: '', tag: flowStore.val.tag } - jobId = await runFlowPreview( - whileLoop ? withWhileLoopIter(args) : args, - newFlow, - $pathStore, - restartedFrom, - undefined, - undefined, - opWorkspace?.() - ) + try { + jobId = await runFlowPreview( + whileLoop ? withWhileLoopIter(args) : args, + newFlow, + $pathStore, + restartedFrom, + undefined, + undefined, + opWorkspace?.() + ) + } catch (err: any) { + // `runFlowPreview` resolves a linked agent's draft first and refuses when it cannot be read. + // Without this the rejection is unhandled and the button just does nothing. + sendUserToast(`Could not run preview: ${err?.body ?? err}`, true) + return + } isRunning = true } diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 96b6eed293..3b860dc3a1 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -294,7 +294,9 @@ untrack(() => { for (const entry of refs ? refs.split('\u0001') : []) { const [moduleId, agentPath] = entry.split('\u0000') - publishLinkedAgentTools(agentPath, ws, scope, moduleId) + // Without the draft: this viewer describes a run that has already happened, and it ran + // the deployed agent. + publishLinkedAgentTools(agentPath, ws, scope, moduleId, false) } }) }) diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index b4e82ced2f..64a318acc7 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -14,6 +14,14 @@ import JobLoader, { type Callbacks } from './JobLoader.svelte' import { getStepHistoryLoaderContext } from './stepHistoryLoader.svelte' import { loadSchemaFromModule } from './flows/flowInfers' + import { + inlineAgentDraft, + loadLinkedAgentDrafts, + normalizeAgentRef, + type LinkedAgentDraft + } from './flows/linkedAgentDrafts' + import { AGENT_FLOW_LOCAL_KEYS } from './flows/agentResourceUtils' + import { sendUserToast } from '$lib/toast' interface Props { mod: FlowModule @@ -138,7 +146,35 @@ } else if (val.type == 'aiagent') { const { schema } = await loadSchemaFromModule(mod, opWs) - const agentVal = val + // A linked step whose agent has an unsaved draft is tested as the draft, the same way the + // whole-flow preview and the agent editor's own test pane run it. `inlineAgentDraft` + // clears `agent` and moves the draft's brain and tools onto the step, so the branches + // below then treat it as a standalone agent. + let draft: LinkedAgentDraft | undefined + if (val.agent) { + const linked = normalizeAgentRef(val.agent) + try { + draft = (await loadLinkedAgentDrafts([linked], opWs)).get(linked) + } catch (err: any) { + // The load refuses when the agent's unsaved changes cannot be read, and this function's + // caller neither awaits nor catches: without this the rejection is unhandled and the + // button appears to do nothing, with the test already marked as started. + sendUserToast(`Could not run test: ${err?.body ?? err}`, true) + // Guarded like every other access to it here: the entry is only created for steps the + // panel is tracking, and this runs on a path where it may never have been. + if (modulesTestStates.states[mod.id]) { + modulesTestStates.states[mod.id].loading = false + } + return + } + } + const agentVal = draft ? inlineAgentDraft(val, draft.args) : val + + // `args` is built from the whole AI agent schema whatever the step is, so on a linked step + // it carries every brain key as undefined even though the form renders only the flow-local + // ones (`flowLocalAgentSchema`). Overlaying those would shadow the brain the draft just + // supplied with nothing, so an inlined step takes only the inputs its form actually offers. + const formKeys = draft ? (AGENT_FLOW_LOCAL_KEYS as readonly string[]) : Object.keys(args) // The test form only covers the schema it was given, and for a standalone agent that may be // the flow-local one (the agent editor shows the brain in its own form, not here). Take the @@ -150,7 +186,7 @@ ? {} : ((agentVal.input_transforms ?? {}) as Record)), ...Object.fromEntries( - Object.keys(args).map((key) => [ + formKeys.map((key) => [ key, { expr: `flow_input.${key}`, diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 89acac79de..3c9eda9467 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -2,7 +2,7 @@ import type { Schema } from '$lib/common' import { ResourceService, WorkspaceService, type Resource, type ResourceType } from '$lib/gen' import { canWrite } from '$lib/utils' - import { createEventDispatcher, untrack } from 'svelte' + import { createEventDispatcher, onDestroy, untrack } from 'svelte' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' @@ -13,7 +13,9 @@ import { getUserExt } from '$lib/user' import type { UserExt } from '$lib/stores' import { UserDraft, draftValuesEqual, type UserDraftHandle } from '$lib/userDraft.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' + import { onUserInput } from '$lib/userDraftEditGate' interface Props { canSave?: boolean @@ -108,6 +110,55 @@ workspaceSpecs.push({ ws, defaultValue }) } + // Gated per workspace until the user puts something into that workspace's + // form (see `onUserInput`): the autosave stays suspended and the deployed + // baseline absorbs whatever the form settles on. A workspace opened ON a + // saved draft keeps its baseline — that divergence is the user's own. + let userEdited: Record = $state({}) + let openedOnDraft: Record = $state({}) + const suspendedWorkspaces = new Set() + + function setGated(ws: string, gated: boolean): void { + if (!initialPath) return + if (gated === suspendedWorkspaces.has(ws)) return + if (gated) { + UserDraft.stopSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.add(ws) + } else { + UserDraft.restartSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.delete(ws) + } + } + + // Nothing counts until this workspace's form is on screen, and while the + // schema is still arriving a precursor alone does not: it would open the gate + // just in time for the schema's materialized values to POST. `Path`, the + // labels and the description render above that skeleton and stay editable + // throughout, so a real value event still counts and keeps the edit. + onUserInput((kind) => { + if (!selected || !(selected in states)) return + if (kind === 'precursor' && loadingSchema) return + userEdited[selected] = true + }) + + $effect(() => { + const wss = Object.keys(states) + const edited = { ...userEdited } + const onDraft = { ...openedOnDraft } + untrack(() => { + // A workspace opened on a saved draft is never suspended — there is no + // phantom to prevent, and a write made while suspended is dropped for + // good. Without `onDraft` here this effect re-suspends it the moment its + // handle appears, undoing the decision made when it was opened. + for (const ws of wss) setGated(ws, !edited[ws] && !onDraft[ws]) + }) + }) + + // `stopSync` must be paired or the key stays unsynced for the session. + onDestroy(() => { + for (const ws of [...suspendedWorkspaces]) setGated(ws, false) + }) + let isValid = $state(true) let jsonError = $state('') let perWsValid: Record = $state({}) @@ -245,6 +296,13 @@ } // Open with the saved draft if present, else the deployed. const s: ResourceState = savedDraftState ?? deployedState + openedOnDraft[ws] = !!savedDraftState + // Gate BEFORE the handle is acquired: `stopSync` queues on a + // not-yet-live entry, and the form can settle before the effect + // above gets a chance to run. Only worth doing when no draft exists + // yet — where one does, there is no phantom to prevent and + // suspending could only drop a write. + if (!savedDraftState) setGated(ws, true) ensureHandle(ws, s) initialStates[ws] = structuredClone(deployedState) // Draft-only paths (`no_deployed`) have no row — saving must @@ -259,6 +317,47 @@ }) }) + /** The schema can only ever write `args`. `path`, `labels`, `description` and + * `wsSpecific` are beyond its reach, so a difference in one of those is the + * user's — whatever event did or didn't reach the gate. Removing a label runs + * a click handler and emits nothing native, and would otherwise be absorbed. */ + function differsOutsideArgs(a: ResourceState, b: ResourceState | undefined): boolean { + return !!b && !draftValuesEqual({ ...a, args: null }, { ...b, args: null }) + } + + // Absorb the form's settling writes into the deployed baseline while the + // selected workspace is gated, so they show up neither as the "unsaved + // changes" banner nor, once `discardIf` reads the baseline, as a draft. + // Only the selected workspace has a form rendered against it. + $effect(() => { + const ws = selected + if (!ws || !initialPath) return + if (userEdited[ws] || openedOnDraft[ws]) return + // `$state.snapshot` deep-reads, so nested `args` mutations re-run this. + const settled = states[ws]?.draft + ? ($state.snapshot(states[ws].draft) as ResourceState) + : undefined + untrack(() => { + if (!settled) return + if (differsOutsideArgs(settled, initialStates[ws])) { + // An edit, not settling. This runs AFTER the write landed, and a write + // made while suspended is swallowed for good (the mirror advances its + // baseline either way), so un-suspend and push the value here rather + // than leaving it to whichever effect happens to run next. + userEdited[ws] = true + setGated(ws, false) + void UserDraftDbSyncer.save({ + workspace: ws, + itemKind: 'resource', + path: initialPath, + value: settled + }) + return + } + if (!draftValuesEqual(settled, initialStates[ws])) initialStates[ws] = settled + }) + }) + // Keep current.path bound to the outer `path` prop for consumers $effect(() => { if (current) path = current.path @@ -292,6 +391,13 @@ } export function discardLocalDraft(): void { if (!selected) return + // Back to the deployed value with nothing of the user's left in it, so + // the gate closes again — otherwise the form settles on the schema's + // values a second time and the discarded draft comes straight back. + // `discard` POSTs the delete itself, so suspending first is safe. + openedOnDraft[selected] = false + userEdited[selected] = false + setGated(selected, true) UserDraft.discard('resource', initialPath ?? '', initialStates[selected], { workspace: selected }) diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 03b410d232..cac8952706 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -84,6 +84,8 @@ initialPath?: string } + let { initialPath }: Props = $props() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -100,25 +102,30 @@ let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean') let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean') - let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema)) + function filterSeeds() { + return { + path: initialPath || undefined, + job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined, + show_future_jobs: showFutureJobsStorage.val === false ? false : undefined + } + } - let { initialPath }: Props = $props() + let filters = useUrlSyncedFilterInstance( + untrack(() => runsFilterSearchbarSchema), + untrack(filterSeeds) + ) + + // `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own + // "Runs" entry, `/runs/` → `/runs`, Back — rewrites the query without remounting, and + // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, + // which never reaches `page.url`, so a filter the user clears stays cleared. + $effect(() => { + page.url.href + untrack(() => filters.seed(filterSeeds())) + }) let batchRerunOptionsIsOpen = $state(false) - // Initialize path filter from route param if provided and not already set via query params - if (untrack(() => initialPath) && !filters.val.path) { - filters.val.path = untrack(() => initialPath) - } - - // Apply persistent toggle values from local storage if URL doesn't specify them - if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) { - filters.val.job_trigger_kind = '!schedule' - } - if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) { - filters.val.show_future_jobs = false - } - // Sync toggle state back to local storage when filters change $effect(() => { if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') { diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index e069fe50b8..393744dbca 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -102,7 +102,7 @@ import type { SavedAndModifiedValue } from './common/confirmationModal/unsavedTypes' import DeployButton from './DeployButton.svelte' import { type Trigger, deployTriggers, handleSelectTriggerFromKind } from './triggers/utils' - import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' + import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte' import { Triggers } from './triggers/triggers.svelte' import type { ScriptBuilderProps } from './script_builder' import WorkerTagSelect from './WorkerTagSelect.svelte' @@ -1149,7 +1149,7 @@ currentValue={script} /> - t.draftConfig)} on:canceled={() => { diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index fdab141d03..f8028aeb1f 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -15,7 +15,7 @@ import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils' import Editor from './Editor.svelte' import { inferArgs, inferAssets, inferAnsibleExecutionMode } from '$lib/infer' - import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + import { injectPartitionArg } from '$lib/scriptEditorSchema' import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow' import WacDiagram from '$lib/components/graph/WacDiagram.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -1074,55 +1074,6 @@ } } - // A `// partitioned` pipeline script is materialized one slice at a time and - // receives the slice as a runtime `partition` arg (the cascade injects it in - // production). It isn't a code parameter, so schema inference doesn't see it — - // surface it in the test form so a partitioned script can be run manually. - function injectPartitionArg( - s: any, - a: Record | undefined, - l: string | undefined, - c: string - ) { - try { - if (l !== 'duckdb' || !s?.properties) return - const part = parsePipelineAnnotations(c).partition - if (!part) return - // Date-based partition kinds render a date / datetime picker; a dynamic - // key is a free-form string. - const format = - part.kind === 'hourly' - ? 'date-time' - : part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly' - ? 'date' - : undefined - if (!s.properties['partition']) { - s.properties['partition'] = { - type: 'string', - ...(format ? { format } : {}), - // ISO output so partition keys sort lexicographically (the date - // picker defaults to dd-MM-yyyy otherwise). - ...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}), - description: - part.kind === 'dynamic' - ? 'Partition key value to materialize.' - : `Partition (${part.kind}) to materialize.` - } - if (Array.isArray(s.order) && !s.order.includes('partition')) { - s.order = ['partition', ...s.order] - } - } - // Pre-fill the *test* arg with the current slice for date kinds — a - // convenience default, kept on the args (not baked into the schema, - // where it would persist to the deployed script and go stale). - if (format && a && (a['partition'] == null || a['partition'] === '')) { - const now = new Date() - a['partition'] = - format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16) - } - } catch (e) {} - } - async function inferModuleSchema() { if (activeModuleTab === null) return try { diff --git a/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts new file mode 100644 index 0000000000..1dec5e9da8 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { renderDbEqualityFilter, renderDbLiteral } from './utils' + +describe('renderDbLiteral', () => { + it('doubles single quotes on every dialect', () => { + expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'") + expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'") + }) + + it('doubles backslashes only where the dialect treats them as escapes', () => { + expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'") + expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'") + expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'") + }) + + it('marks SQL Server strings as Unicode constants', () => { + expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'") + }) + + it('renders numbers and booleans without quotes', () => { + expect(renderDbLiteral(42, 'postgresql')).toBe('42') + expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE') + expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1') + }) + + it('has no literal for values that cannot be compared safely', () => { + expect(renderDbLiteral(null, 'postgresql')).toBeUndefined() + expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined() + expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined() + }) +}) + +describe('renderDbEqualityFilter', () => { + it('quotes the identifier per dialect', () => { + expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'") + expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined() + }) + + it('doubles a delimiter embedded in the identifier', () => { + expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`) + expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1') + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index bf81ec07bb..c0477e8408 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -333,25 +333,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string { .join(', ') } +/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an + * embedded delimiter. */ export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string { switch (dbType) { case 'postgresql': - return `"${identifier}"` // PostgreSQL uses double quotes for identifiers - case 'ms_sql_server': - return `[${identifier}]` // MSSQL uses square brackets for identifiers - case 'mysql': - return `\`${identifier}\`` // MySQL uses backticks case 'snowflake': - return `"${identifier}"` // Snowflake uses double quotes for identifiers - case 'bigquery': - return `\`${identifier}\`` // BigQuery uses backticks case 'duckdb': - return `"${identifier}"` // DuckDB uses double quotes for identifiers + return `"${identifier.replace(/"/g, '""')}"` + case 'ms_sql_server': + return `[${identifier.replace(/]/g, ']]')}]` + case 'mysql': + case 'bigquery': + return `\`${identifier.replace(/`/g, '``')}\`` default: throw new Error('Unsupported database type: ' + dbType) } } +/** Renders a cell value as a SQL literal. Returns undefined for values that + * have no safe literal form (null, objects, non-finite numbers). */ +export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined { + if (value === null || value === undefined) return undefined + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') { + if (dbType === 'ms_sql_server') return value ? '1' : '0' + return value ? 'TRUE' : 'FALSE' + } + if (typeof value !== 'string') return undefined + let escaped = value.replace(/'/g, "''") + // MySQL, Snowflake and BigQuery treat a backslash inside a string literal as + // an escape character. + if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') { + escaped = escaped.replace(/\\/g, '\\\\') + } + // A plain constant is varchar on SQL Server and goes through the database + // code page; the N prefix keeps it Unicode against nvarchar columns. + return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'` +} + +/** `"column" = ` predicate, or undefined when the value can't be + * rendered as a literal. */ +export function renderDbEqualityFilter( + column: string, + value: unknown, + dbType: DbType +): string | undefined { + const literal = renderDbLiteral(value, dbType) + if (literal === undefined) return undefined + return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}` +} + export function getLanguageByResourceType(name: string): ScriptLang { const language = { postgresql: 'postgresql', diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index d5839c6766..99ea07c8fd 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -22,19 +22,22 @@ let { workspace, - path, - editHref + path }: { workspace: string path: string - /** Where the Edit button points (low-code vs raw editor). */ - editHref: string } = $props() let app: any = $state(undefined) let notExists = $state(false) let noPermission = $state(false) let canWriteApp = $state(false) + /** Raw vs low-code, read from the app itself rather than from the route: + * both kinds render here and either route serves either kind (links to a raw + * app point at /apps/get all over the app), so only the app can say which + * editor the Edit button must open. */ + let isRawApp = $state(false) + let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined // The opaque iframe loads the dedicated cookieless, chrome-less viewer route. @@ -103,11 +106,14 @@ } } - // Edit button: determine write access on this real-origin page (cookie). + // Edit button: determine write access and which editor to open on this + // real-origin page (cookie). The sandboxed low-code app never loads on this + // page (it loads inside the opaque iframe), so `app` can't be the source. async function loadPerms() { try { const lite: any = await AppService.getAppLiteByPath({ workspace, path }) canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore) + isRawApp = !!lite?.raw_app } catch (_) { canWriteApp = false } diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 2c5e5fd97e..005e3a3ffb 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -36,7 +36,8 @@ type ColumnLineage, type PipelineAnnotations } from './parsePipelineAnnotations' - import ColumnLineageTrace from './ColumnLineageTrace.svelte' + import ColumnTraceSection from './ColumnTraceSection.svelte' + import DbtColumnList from './DbtColumnList.svelte' import { extractDraftMacros } from './resolveGraph' import { assetColumnNodes, type ColumnLineageGraph } from './columnLineageGraph' import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte' @@ -156,6 +157,12 @@ // resolved graph). Drives the transitive column-lineage trace shown for a // selected materialized asset. selectionColumnGraph?: ColumnLineageGraph + /** That graph still being fetched — a dbt relation's lineage is a request + * of its own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** The lineage reaches past what the graph holds: the API cut it at the + * part nearest the selection. */ + selectionColumnTruncated?: boolean /** dbt provenance of the selected relation, when a dbt project * materializes it — carries the model's own SQL. */ selectionDbt?: DbtAssetProvenance @@ -289,6 +296,8 @@ onScriptRemoved, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -446,6 +455,19 @@ return scripts.length === 1 ? `${scripts[0].path}__dbt/${file}` : file }) + // The two things a dbt relation can show besides its SQL, and what decides + // whether the panel opens at all for one that has none: the columns the model + // produces, and the trace they sit in. A share-link viewer gets neither — + // both are gated on reading the project, like the SQL. + let selectionDbtHasColumns = $derived( + !!selectionDbt?.column_schema?.length || Object.keys(selectionDbt?.columns ?? {}).length > 0 + ) + let selectionColumnNodes = $derived( + selection?.kind === 'asset' && selectionColumnGraph + ? assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path) + : [] + ) + // Bound from ScriptEditor — populated by inferAssets on every code // change. Forwarded to the page so the canvas can re-derive write // edges as the user edits the body (e.g. renaming a CREATE TABLE @@ -1221,22 +1243,20 @@ {/if} - {#if selectionColumnGraph && assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path).length > 0} -
- -
- {/if} +
{/key} - {:else if selectionDbt?.raw_code} + {:else if selectionDbt && (selectionDbt.raw_code || selectionDbtHasColumns || selectionColumnNodes.length > 0 || selectionColumnLoading)} + {#if selectionDbtHasColumns} +
+ +
+ {/if} + + {#if selectionDbt.raw_code} +
+ +
+ {/if} {:else}
diff --git a/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte new file mode 100644 index 0000000000..0422b88cc8 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte @@ -0,0 +1,50 @@ + + +{#if loading && nodes.length === 0} +
+ + Loading column lineage +
+{:else if graph && nodes.length > 0} +
+ + {#if truncated} +
+ Showing the part of the trace nearest this relation. The lineage reaches further than one + view can draw. +
+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte new file mode 100644 index 0000000000..a1c769d9af --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte @@ -0,0 +1,50 @@ + + +{#if columns.length > 0} +
+
{analyzed ? 'columns' : 'columns declared'}
+
+ {#each columns as col (col.name)} +
+ {col.name} + {#if col.type} + {col.type} + {/if} + {col.description} +
+ {/each} +
+ + {#if !analyzed} +
+ Declared metadata. Set `column_lineage: true` in the descriptor for the real column schema, + typed and in the order the model produces it. +
+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 3439fa5ad5..721fafb6be 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -17,7 +17,9 @@ AssetGraphResponse, AssetGraphSelection, NativeTriggerKind, - PipelineMode, DbtAssetProvenance } from './types' + PipelineMode, + DbtAssetProvenance + } from './types' import type { AssetKind, Script, ScriptLang } from '$lib/gen' import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte' import type { PipelineOutputKind } from './pipelineTemplates' @@ -76,6 +78,8 @@ localScriptsVersion, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -180,8 +184,13 @@ * the selected node's source on live-reload. */ localScriptsVersion?: unknown selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> - /** Transitive column-lineage trace for a selected ducklake asset (route page). */ + /** Transitive column-lineage trace for the selected asset (route page). */ selectionColumnGraph?: ColumnLineageGraph + /** That trace still being fetched — a dbt relation's is a request of its + * own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** That trace cut at the part nearest the selection. */ + selectionColumnTruncated?: boolean /** dbt provenance of the selected relation — carries its SQL. */ selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean @@ -514,6 +523,8 @@ selection={activeDraft ? undefined : editor.selection} selectionProducers={activeDraft ? [] : selectionProducers} {selectionColumnGraph} + {selectionColumnLoading} + {selectionColumnTruncated} {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts index 6842dcfd86..16c8713c5c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import type { AssetGraphResponse } from './types' import { buildColumnGraph, + buildDbtColumnGraph, colNodeId, + mergeColumnGraphs, + type ColumnLineageGraph, traceColumn, connectedComponent, assetColumnNodes, @@ -120,6 +123,79 @@ describe('buildColumnGraph', () => { }) }) +describe('buildDbtColumnGraph', () => { + it('takes the direct kinds and drops any other', () => { + // `scan` means the column was read to produce the ROW — a join key, a + // predicate, a `group by` — so it reaches every output column of its model + // and is not what a column trace means. The server filters it out; this + // filters again, because the kind set is the engine's and an unknown one + // must not become an edge the trace calls data flow. + const g = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw_name', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'mod' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'id', + kind: 'copy' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'scan' + } + ]) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'clean_name'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw_name')]) + ) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'id'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'id')]) + ) + }) +}) + +describe('mergeColumnGraphs', () => { + it('chains a dbt column into what a producer derives from it', () => { + // The two halves arrive separately — the producer's from the asset graph, + // dbt's from its own request — and meet at the dbt node a `// column` + // annotation names. A trace has to cross that, or a dbt selection stops + // before the script consuming it. + const dbt = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw', + to_asset_path: 'main/s/mart', + to_column: 'clean', + kind: 'copy' + } + ]) + const producer: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() + } + const src = colNodeId('dbt', 'main/s/mart', 'clean') + const out = colNodeId('ducklake', 'wh/report', 'total') + producer.nodes.set(src, { kind: 'dbt', path: 'main/s/mart', column: 'clean' }) + producer.nodes.set(out, { kind: 'ducklake', path: 'wh/report', column: 'total' }) + producer.up.set(out, new Set([src])) + producer.down.set(src, new Set([out])) + + const merged = mergeColumnGraphs(dbt, producer) + expect(traceColumn(colNodeId('dbt', 'main/s/stg', 'raw'), merged)).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw'), src, out]) + ) + }) +}) + describe('traceColumn', () => { it('returns the full upstream + downstream impact set of a source column', () => { const g = buildColumnGraph(chainGraph()) diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts index 50fa1c4e7b..54837753e1 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts @@ -1,6 +1,11 @@ -import type { AssetKind } from '$lib/gen' +import type { AssetKind, DbtColumnLineage } from '$lib/gen' import type { AssetGraphResponse } from './types' +// One column-to-column edge of a dbt project's static analysis, as the API +// serves it. Taken from the generated client rather than restated: unlike the +// asset graph, this response is fetched through it. +export type DbtColumnEdge = DbtColumnLineage['edges'][number] + // A node in the column-level lineage graph: one column of one asset. export type ColumnNode = { kind: AssetKind; path: string; column: string } export type ColumnNodeId = string @@ -23,6 +28,21 @@ export type ColumnLineageGraph = { down: Map> } +export const EMPTY_COLUMN_GRAPH: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() +} + +// Direct value flow, as dbt's static analysis labels it: `copy` passes a column +// through, `mod` transforms it. The API serves only those two — the third kind, +// `scan`, means the column was read to produce the ROW rather than the value (a +// join key, a `where` predicate, a `group by`), so it reaches EVERY output +// column of the model and would draw the diagram as a complete bipartite graph. +// Filtered here as well so a kind the engine invents cannot silently become an +// edge the trace claims is data flow. +const DIRECT_DBT_LINEAGE = new Set(['copy', 'mod']) + // Build the column graph from a resolved asset graph. A producer's // `column_lineage` describes the columns of the asset it materializes; that // output asset is the ducklake target it writes (v1 materialize target), found @@ -89,6 +109,59 @@ export function buildColumnGraph(graph: AssetGraphResponse): ColumnLineageGraph return { nodes, up, down } } +// The same graph, from dbt's own column lineage. dbt arrives already resolved to +// two relations rather than anchored to a producer, and the API serves only the +// direct kinds, so this is a straight edge list. +export function buildDbtColumnGraph(edges: DbtColumnEdge[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + const addNode = (n: ColumnNode): ColumnNodeId => { + const id = colNodeId(n.kind, n.path, n.column) + if (!nodes.has(id)) nodes.set(id, n) + return id + } + for (const e of edges) { + // Belt and braces: the API filters to `copy`/`mod`, and a kind an engine + // invents must not silently become an edge the trace calls data flow. + if (!DIRECT_DBT_LINEAGE.has(e.kind)) continue + const src = addNode({ kind: 'dbt', path: e.from_asset_path, column: e.from_column }) + const out = addNode({ kind: 'dbt', path: e.to_asset_path, column: e.to_column }) + if (src === out) continue + ;(up.get(out) ?? up.set(out, new Set()).get(out)!).add(src) + ;(down.get(src) ?? down.set(src, new Set()).get(src)!).add(out) + } + return { nodes, up, down } +} + +// One graph out of several, so a trace crosses the boundary between them. +// +// The two halves reach each other through shared node ids: a producer's +// `// column out <- dbt://wh/schema/model.col` puts a `('dbt', path, column)` +// node in the producer graph under the same `colNodeId` the dbt lineage mints +// for it, so the union chains a dbt model's columns into the script that +// consumes them and on into what that script writes. Kept separate up to here +// because they are fetched separately — the producer half rides on the asset +// graph, the dbt half is asked for per selection. +export function mergeColumnGraphs(...graphs: ColumnLineageGraph[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + for (const g of graphs) { + for (const [id, n] of g.nodes) if (!nodes.has(id)) nodes.set(id, n) + for (const [dir, into] of [ + [g.up, up], + [g.down, down] + ] as const) { + for (const [id, adj] of dir) { + const target = into.get(id) ?? into.set(id, new Set()).get(id)! + for (const m of adj) target.add(m) + } + } + } + return { nodes, up, down } +} + // Every node reachable from `start` by following `adj` (transitive closure, // excluding `start` itself). Iterative to avoid deep-recursion limits. function reach(start: ColumnNodeId, adj: Map>): Set { diff --git a/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts new file mode 100644 index 0000000000..189ba05b5d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts @@ -0,0 +1,120 @@ +import { AssetService, JobService, type DbtColumnLineage } from '$lib/gen' +import { + buildDbtColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph +} from './columnLineageGraph' + +/** Which stored dbt graph a view is drawing. A job — a run's snapshot, or the + * editor's parse of its own buffer — is asked through the job route, the only + * way to reach a graph that names no deployed version; otherwise the deployed + * version by hash, or the current one when there is no hash. */ +export type DbtGraphPin = { jobId?: string; scriptHash?: string | number } + +/** What a selection's dbt column lineage is doing right now. `loading` is + * separate because a project still being fetched and one that never asked for + * the analysis pass are the same empty graph otherwise. */ +export type DbtColumnLineageState = { + readonly graph: ColumnLineageGraph + readonly loading: boolean + /** The component reaches past what `graph` holds — the API cut it at the + * part nearest the selection. */ + readonly truncated: boolean +} + +function fetchLineage( + workspace: string, + assetPaths: string[], + pin: DbtGraphPin | undefined +): Promise { + return pin?.jobId + ? JobService.getDbtRunColumnLineage({ workspace, id: pin.jobId, assetPath: assetPaths }) + : AssetService.getDbtColumnLineage({ + workspace, + assetPath: assetPaths, + dbtScriptHash: pin?.scriptHash != undefined ? String(pin.scriptHash) : undefined + }) +} + +/** Follow the selection, fetching the dbt column lineage it reaches. + * + * One request per selection, whatever it reaches: the API takes every relation + * at once and walks out from all of them, so there is no partial answer to hold + * on to between selections and nothing to go stale behind a redeploy. + * + * Per selection rather than off the graph response: the graph is folder-wide + * and a run page polls it, while this is drawn for one selection. It also means + * the request is never made for a project that did not opt into the analysis + * pass — the pane simply never shows the section. + */ +export function useDbtColumnLineage(args: { + workspace: () => string | undefined + /** The dbt relations to expand. The selection itself when it is one; for a + * selection of another kind, every dbt relation its own lineage reaches — + * a ducklake table can be derived from several, and expanding only the + * first would leave the rest as leaves. */ + assetPaths: () => string[] + /** The graph on screen, so the lineage describes the same project. */ + pin?: () => DbtGraphPin | undefined +}): DbtColumnLineageState { + let graph = $state(EMPTY_COLUMN_GRAPH) + let loading = $state(false) + let truncated = $state(false) + + // The question the state in hand answers, and a counter deciding which answer + // is still wanted. Neither is a cache of edges: the API returns a whole + // component, so an answer is either the current selection's or nothing. + let asked: string | undefined = undefined + let latest = 0 + + $effect(() => { + const workspace = args.workspace() + const paths = [...new Set(args.assetPaths())].sort() + const pin = args.pin?.() + const question = JSON.stringify([workspace, pin?.jobId, pin?.scriptHash, paths]) + // A selection re-derived from a graph that polled is the same question. Not + // asking it again is what keeps a run page from refetching a component's + // worth of edges every poll to redraw what is already on screen. + if (question === asked) return + asked = question + const id = ++latest + if (!workspace || paths.length === 0) { + graph = EMPTY_COLUMN_GRAPH + truncated = false + loading = false + return + } + loading = true + fetchLineage(workspace, paths, pin).then( + (r) => { + if (id !== latest) return + graph = buildDbtColumnGraph(r?.edges ?? []) + truncated = r?.truncated ?? false + loading = false + }, + // Lineage annotates a graph that renders without it, so a failed fetch + // leaves that branch unexpanded rather than putting an error over the + // model. Not retried on its own: the effect reruns whenever the canvas + // does, and a failing endpoint would then be asked once per redraw. + // Selecting another node and back asks again. + () => { + if (id !== latest) return + graph = EMPTY_COLUMN_GRAPH + truncated = false + loading = false + } + ) + }) + + return { + get graph() { + return graph + }, + get loading() { + return loading + }, + get truncated() { + return truncated + } + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index de00aef2be..c57687d78b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -319,6 +319,18 @@ describe('resolveGraph', () => { expect(assetTrigKeys(r, 'f/x/open')).toEqual(['ducklake:main.orders']) }) + // A relation a native `// materialize manual dbt://…` script writes wakes its + // subscribers, so hiding the edge would leave the author's own annotation off + // the canvas. The deploy refuses the ones that cannot fire. + it('draws an explicit dbt:// subscription as an unsaved trigger overlay', () => { + const liveAnnotations = { + scriptPath: 'f/x/open', + annotations: ann({ triggerAssets: [{ kind: 'dbt', path: 'main/analytics/orders' }] }) + } + const r = resolveGraph(input({ liveAnnotations })) + expect(assetTrigKeys(r, 'f/x/open')).toEqual(['dbt:main/analytics/orders']) + }) + it('open-script live annotations add unsaved triggers, deduped vs persisted', () => { const base = baseGraph({ triggers: [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index 867fa6f036..5c8aa697a4 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -77,23 +77,15 @@ function persistedNativeKinds(base: AssetGraphResponse, path: string): Set = new Set(['ducklake', 's3object']) -/** Whether a subscription on this kind can ever fire once deployed. - * - * A `dbt://` one cannot: dbt is the only producer of a warehouse relation and - * a dbt run does not dispatch, so the deploy refuses `// on dbt://…` outright - * (`scripts.rs`). The editor must not draw an arrow the deploy will reject — - * applied to the EXPLICIT overlays; auto-derivation is already scoped by - * `AUTO_TRIGGER_KINDS`. Parsing is left alone so the Rust-parity test still - * compares like for like. */ -function canTrigger(kind: AssetKind): boolean { - return kind !== 'dbt' -} - /** `kind:path` refs of a script's `// materialize` write target(s) (base + * the scd2 `_current` companion), which the body `SELECT` doesn't express. */ function materializeWriteRefs(parsed: PipelineAnnotations): string[] { @@ -381,7 +373,7 @@ function makeContext(input: ResolveGraphInput): ResolveContext { const liveRefKeys = new Set() if (openIsSavedEdit) { if (liveAnnotations.scriptPath === openPath) { - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) + for (const a of liveAnnotations.annotations.triggerAssets) liveRefKeys.add(`${a.kind}:${a.path}`) // The `// materialize ` target is a declared *output*, but it // lives in an annotation (not the SQL body), so neither triggerAssets @@ -625,7 +617,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // stable when the user clicks off this draft. Live annotations // (below) take over for the currently-open draft so keystroke // edits still update in real time. - for (const a of parsed.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of parsed.triggerAssets) { extraTriggers.push({ trigger_kind: 'asset', asset_kind: a.kind, @@ -704,7 +696,7 @@ function applyLiveBufferOverlay(acc: Accumulator, input: ResolveGraphInput, ctx: for (let i = extraTriggers.length - 1; i >= 0; i--) { if (extraTriggers[i].runnable_path === livePath) extraTriggers.splice(i, 1) } - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of liveAnnotations.annotations.triggerAssets) { const key = `${a.kind}:${a.path}` if (assetKeys.has(key)) continue extraTriggers.push({ diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 4ada1feec3..023f71e218 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -36,9 +36,14 @@ export interface DbtAssetProvenance { tags?: string[] description?: string data_tests?: DbtDataTest[] - /** Declared column metadata (name -> description). NOT column lineage: - * `manifest.json` carries none (docs/dbt-runtime.md, decision 14). */ + /** Declared column metadata (name -> description): what `manifest.json` + * carries, which is only the columns an author wrote down. */ columns?: Record + /** Every column of the relation, typed and in the order the model produces + * them, from the engine's static analysis. Present only for a project that + * opted into it (`column_lineage: true`); `manifest.json` has no such + * thing. Lockstep with Rust `DbtAssetProvenance.column_schema`. */ + column_schema?: { name: string; type?: string }[] /** A source's declared freshness policy. */ freshness?: unknown /** The model's SQL as written — the transform behind the node. Read-only: diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 23531d71f2..93367a725e 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -96,10 +96,11 @@ export function formatAssetKind(asset: { case 'volume': return 'Volume' case 'dbt': - // The SCHEME says dbt because dbt is the only thing that creates one; - // the PATH stays the relation, so a mart one project builds and the - // `source` the next reads land on one node — their dbt `unique_id`s - // differ where the relation does not (docs/dbt-runtime.md, decision 11). + // The SCHEME says dbt because dbt is what derives these relations; the + // PATH stays the relation, so a mart one project builds, the `source` + // the next reads, and a native `// materialize manual dbt://…` writer + // land on one node — their dbt `unique_id`s differ where the relation + // does not (docs/dbt-runtime.md, decision 11). return 'dbt table' } } diff --git a/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte new file mode 100644 index 0000000000..dadf814a80 --- /dev/null +++ b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte @@ -0,0 +1,323 @@ + + + dispatch('canceled')} + on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })} +> +
+ {#if draftTriggers.length > 0} +
+
+ {`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`} +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + {#each draftTriggers as trigger} + {@const SvelteComponent = triggerIconMap[trigger.type]} + {@const permission = checkSavePermissions(trigger)} + {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} + + + + + + {/each} + + + + + {/if} + + {#if draftAgents.length > 0} +
+
+ Saved agents this flow uses have unsaved changes. Select which ones to deploy with the + flow. An agent kept as a draft stays editable, and the flow runs the agent as currently + deployed. +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + + {#each draftAgents as agent (agent.path)} + {@const permission = checkAgentPermissions(agent)} + {@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)} + + + + + + + + {/each} + + + + + {/if} + + diff --git a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte deleted file mode 100644 index cfebae0e9f..0000000000 --- a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte +++ /dev/null @@ -1,168 +0,0 @@ - - - dispatch('canceled')} - on:confirmed={() => dispatch('confirmed', { selectedTriggers })} -> -
-
- {`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed - draft triggers will be permanently deleted.`} -
- -
5 ? 'h-[300px]' : ''}> - -
- - - - - - - {#each draftTriggers as trigger} - {@const SvelteComponent = triggerIconMap[trigger.type]} - {@const permission = checkSavePermissions(trigger)} - {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} - - - - - - {/each} - - {#if draftTriggers.length === 0} - - - - {/if} - - - - - diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index 1e885fa304..e93017b095 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -3,6 +3,7 @@ import { ExternalLink, PanelRight } from 'lucide-svelte' import { Button } from '$lib/components/common' import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import { newTabModifier } from '$lib/attachments/newTabModifier.svelte' import { hasToolDisplayActionHandler, runToolDisplayAction @@ -44,6 +45,8 @@ const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + const modifier = newTabModifier() + const hint = $derived( previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab` ) @@ -67,7 +70,11 @@ {#if href} {#if wmKind} - + + - {#if previewAction} + + {#if previewAction && !modifier.held} {:else} diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts index 7583c1b356..f696b1c869 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -338,8 +338,8 @@ export function getPipelinePromptSection(ctx: PipelineContext): string { Data Pipeline editor (ACTIVE): - The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. - Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). -- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`, \`// measure = [where ]\`, \`// dimension = \`. -- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
Triggers to deploy
+
+
+ + {#if trigger.isPrimary} + + {/if} +
+ +
+ +
+
+ 		+ {#if permission === 'deploy'} +
+ toggleTrigger(trigger, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission === 'admin-only'} + Admin only + {:else if permission === 'invalid-config'} + Invalid config + {/if} +
Agents to deploy
+
+ +
+
+ + + {agent.path} + + {#if agent.noDeployed} + Never deployed + {/if} +
+ {#if agent.noDeployed && !isSelectedAgent} + + + Never deployed, so the flow will not run until this agent is deployed. + + {/if} +
+
+ 		+ {#if permission.state === 'deploy'} +
+ toggleAgent(agent, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission.state === 'read-only'} + + Read-only + + {:else} + Invalid config + {/if} +
Triggers to deploy
-
-
- - {#if trigger.isPrimary} - - {/if} -
-
- -
-
- 		- {#if permission === 'deploy'} -
- toggleTrigger(trigger, e.detail)} - > - {#snippet children({ item })} - - - {/snippet} - -
- {:else if permission === 'admin-only'} - Admin only - {:else if permission === 'invalid-config'} - Invalid config - {/if} -
- No draft triggers found -
\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` target), \`// measure = [where ]\`, \`// dimension = \`. +- \`materialize\` (the managed output): a managed \`// materialize ducklake:///
\` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. The \`dbt://\` target below is the opposite: the node writes its own DDL and none of the write strategies apply to it. IMPORTANT: a MANAGED \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects a \`ducklake://\` target on any other language. For a \`python3\`/\`bun\`/\`postgresql\` node writing the lake, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. The one target any language BUT DBT'S OWN may declare (a dbt project's writes come from its manifest, so \`// materialize\` on a dbt script is rejected at deploy) is a WAREHOUSE RELATION: \`// materialize manual dbt:////\`, with \`\` a warehouse the workspace configures under Settings → dbt. \`manual\` is its only mode — nothing generates warehouse DDL, so the node issues its own write and the annotation records the outcome. Use it on an ingestion node a dbt project reads as a \`source\`: the declared relation and the dbt model become ONE graph node, and a downstream \`// on dbt:////\` fires when that node completes. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". - \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it. - Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. - Every node of this pipeline lives at \`f/${ctx.folder}/\` — \`${ctx.folder}\` is the folder name and \`f/\` is the owner prefix every workspace path carries, so write it exactly once (never \`f/f/…\`, and never a bare \`\`). diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index a89abf4374..5005cc931c 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -14,6 +14,7 @@ import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' import { buildTableEditorValues, + type TableEditorForeignKey, type TableEditorValues } from './apps/components/display/dbtable/tableEditor' import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable' @@ -250,6 +251,10 @@ export type IDbSchemaOps = { schema?: string colDefs: TableMetadata }) => Promise + onFetchForeignKeys: (params: { + table: string + schema?: string + }) => Promise } /** Thrown by a schema op when the user declines the out-of-order run warning. @@ -396,6 +401,48 @@ export function dbSchemaOpsWithPreviewScripts({ } } + /** Resolves to [] when the database has no foreign key introspection + * (BigQuery) or the query fails: callers treat foreign keys as optional. */ + async function fetchForeignKeys({ + table, + schema + }: { + table: string + schema?: string + }): Promise { + if (dbType === 'bigquery') return [] + try { + const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) + const fkResult = await runScriptAndPollResult({ + workspace, + requestBody: { args: dbArg, content: fkContent, language, tag } + }) + + let rawForeignKeys: RawForeignKey[] + if (dbType === 'snowflake') { + rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) + } else { + rawForeignKeys = fkResult as RawForeignKey[] + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + rawForeignKeys = rawForeignKeys.map((fk) => { + const lowerFk: any = {} + Object.keys(fk).forEach((key) => { + lowerFk[key.toLowerCase()] = fk[key] + }) + return lowerFk + }) + } + } + + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + return transformForeignKeys(rawForeignKeys) + } + } catch (e) { + console.warn('Failed to fetch foreign keys:', e) + } + return [] + } + return { onDelete: async ({ tableKey, schema }) => { const content = makeMarker('DROP_TABLE', { table: tableKey, schema }) @@ -454,44 +501,11 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { - let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] = - [] + const foreignKeys = await fetchForeignKeys({ table, schema }) let pk_constraint_name: string | undefined - // Fetch foreign keys (not supported for BigQuery) - if (dbType !== 'bigquery') { - try { - const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) - const fkResult = await runScriptAndPollResult({ - workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } - }) - - let rawForeignKeys: RawForeignKey[] - if (dbType === 'snowflake') { - rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) - } else { - rawForeignKeys = fkResult as RawForeignKey[] - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - rawForeignKeys = rawForeignKeys.map((fk) => { - const lowerFk: any = {} - Object.keys(fk).forEach((key) => { - lowerFk[key.toLowerCase()] = fk[key] - }) - return lowerFk - }) - } - } - - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - foreignKeys = transformForeignKeys(rawForeignKeys) - } - } catch (e) { - console.warn('Failed to fetch foreign keys:', e) - } - } - // Fetch primary key constraint name (not supported for BigQuery/MySQL) if (dbType !== 'bigquery' && dbType !== 'mysql') { try { diff --git a/frontend/src/lib/components/dbt/DbtEditor.svelte b/frontend/src/lib/components/dbt/DbtEditor.svelte index f12e510c61..469536f305 100644 --- a/frontend/src/lib/components/dbt/DbtEditor.svelte +++ b/frontend/src/lib/components/dbt/DbtEditor.svelte @@ -8,7 +8,7 @@ // single file, so the arguments, the run and the graph are all the project's // whichever file happens to be open. import { untrack } from 'svelte' - import { createEventDispatcher, onDestroy } from 'svelte' + import { createEventDispatcher, onDestroy, onMount } from 'svelte' import type { Schema, SupportedLanguage } from '$lib/common' import type { Preview, ScriptModule } from '$lib/gen' import { workspaceStore } from '$lib/stores' @@ -31,6 +31,15 @@ AssetGraphNodeData, DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import { + useDbtColumnLineage, + type DbtGraphPin + } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + EMPTY_COLUMN_GRAPH, + mergeColumnGraphs, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { DBT_DESCRIPTOR, DBT_MODULE_EXTENSIONS, @@ -97,6 +106,17 @@ } }) + // The stored schema can predate the parser (a CLI push, an older version), and + // the autosave baseline is taken as the mounted editor holds it + // (`schemaAsEditorMounts`), so the descriptor is re-inferred on mount rather + // than only on its first edit — otherwise the two only agree once edited. + onMount(async () => { + await inferSchema(code) + // Same single retry as ScriptEditor: the first parse can lose a transient + // wasm init race, and the baseline (`schemaAsEditorMounts`) retries too. + if (!validDescriptor && code) await inferSchema(code) + }) + function flushOpenFile() { if (openFile !== null && modules?.[openFile]) { modules[openFile] = { ...modules[openFile], content: editorCode } @@ -212,6 +232,28 @@ // the deployed graph, which previews by version instead. Either way the rows // come from the project whose SQL is displayed above them. let selectedBuffer = $state(undefined) + // Which graph the selection came from, so the lineage fetched below is the + // selected node's own project rather than whatever is deployed. + let selectionPin = $state(undefined) + // The selected model's column lineage, fetched on selection. Its own request + // rather than a field on the graph: only a project that opted into the + // analysis pass has any, and it is drawn for one model at a time. + const columnLineage = useDbtColumnLineage({ + workspace: () => opWs, + assetPaths: () => { + const path = selectedDbt ? selectedAsset?.path : undefined + return path ? [path] : [] + }, + pin: () => selectionPin + }) + // What the scripts around this project declare about its columns, off the + // same graph response the canvas drew. Merged rather than chosen between: a + // model's column and the ducklake column a script derives from it are one + // chain, and the trace has to cross that boundary. + let selectionProducerColumns = $state(EMPTY_COLUMN_GRAPH) + let selectionColumnGraph = $derived( + mergeColumnGraphs(columnLineage.graph, selectionProducerColumns) + ) let jobLoader: JobLoader | undefined = $state(undefined) let testJob: any = $state(undefined) @@ -514,10 +556,12 @@ testRunning={testIsLoading} testResult={testJob?.result} selection={graphSelection} - onSelect={(sel, dbt, buffer) => { + onSelect={(sel, dbt, buffer, pin, producerColumns) => { graphSelection = sel selectedDbt = dbt selectedBuffer = buffer + selectionPin = pin + selectionProducerColumns = producerColumns }} /> @@ -539,6 +583,9 @@ {args} fileInBundle={!!selectedDbt.original_file_path && !!modules?.[selectedDbt.original_file_path]} + columnGraph={selectionColumnGraph} + columnLoading={columnLineage.loading} + columnTruncated={columnLineage.truncated} onOpenFile={open} onClose={() => (graphSelection = undefined)} /> diff --git a/frontend/src/lib/components/dbt/DbtModelDetails.svelte b/frontend/src/lib/components/dbt/DbtModelDetails.svelte index 2d25dbdc00..d3dd25d04a 100644 --- a/frontend/src/lib/components/dbt/DbtModelDetails.svelte +++ b/frontend/src/lib/components/dbt/DbtModelDetails.svelte @@ -12,6 +12,9 @@ import { ClipboardCopy, Code2, FileCode2, Loader2, TableProperties, X } from 'lucide-svelte' import { copyToClipboard } from '$lib/utils' import type { DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import ColumnTraceSection from '$lib/components/assets/AssetGraph/ColumnTraceSection.svelte' + import DbtColumnList from '$lib/components/assets/AssetGraph/DbtColumnList.svelte' + import type { ColumnLineageGraph } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { previewDbtRows, type DbtPreview, type DbtPreviewBuffer } from './previewRows' import { nodeSelector } from './parseDbtRun' @@ -34,6 +37,12 @@ args, /** Whether this model's file is in the bundle being edited. */ fileInBundle = false, + /** The project's column-level lineage, when the descriptor asked for it. + * Fetched for this relation against the same graph the canvas draws, so + * the trace and the nodes above it describe one parse. */ + columnGraph, + columnLoading = false, + columnTruncated = false, onOpenFile, onClose }: { @@ -45,6 +54,9 @@ buffer?: DbtPreviewBuffer args?: Record fileInBundle?: boolean + columnGraph?: ColumnLineageGraph + columnLoading?: boolean + columnTruncated?: boolean onOpenFile?: (path: string) => void onClose?: () => void } = $props() @@ -111,7 +123,9 @@ return typeof v === 'object' ? JSON.stringify(v) : String(v) } - let columns = $derived(Object.entries(dbt.columns ?? {})) + let hasColumns = $derived( + !!dbt.column_schema?.length || Object.keys(dbt.columns ?? {}).length > 0 + ) // `dbt show` SELECTs from the node's own relation and the worker intersects // the selector with `resource_type:model`, so offering it on a seed, snapshot // or source only ever produces a failed job. @@ -202,15 +216,15 @@ {#if stalePlaceholders}
- The run arguments have changed since this graph was parsed, so these rows need not - describe the models on screen — arguments reach schemas, aliases and which models exist - at all. Refresh the models to draw and preview them under the current ones. + The run arguments have changed since this graph was parsed, so these rows need not describe + the models on screen — arguments reach schemas, aliases and which models exist at all. Refresh + the models to draw and preview them under the current ones.
{:else if staleVars}
- The run form's vars have changed since this graph was parsed. Rows are previewed under - the vars it was parsed with, so they still describe the models on screen — refresh the - models to draw and preview them under the current ones. + The run form's vars have changed since this graph was parsed. Rows are previewed under the + vars it was parsed with, so they still describe the models on screen — refresh the models to + draw and preview them under the current ones.
{/if} @@ -234,26 +248,9 @@ {/if} - {#if columns.length > 0 || (dbt.data_tests?.length ?? 0) > 0} + {#if hasColumns || (dbt.data_tests?.length ?? 0) > 0}
- {#if columns.length > 0} -
-
columns declared
-
- {#each columns as [name, desc] (name)} -
- {name} - {desc} -
- {/each} -
- -
- Declared metadata — dbt reports no column-level lineage. -
-
- {/if} + {#if (dbt.data_tests?.length ?? 0) > 0}
tests
@@ -269,6 +266,15 @@
{/if} + + {#if showRows && preview} {#if 'error' in preview}
{preview.error}
diff --git a/frontend/src/lib/components/dbt/DbtModelGraph.svelte b/frontend/src/lib/components/dbt/DbtModelGraph.svelte index e6d948d926..bd7f42a9e7 100644 --- a/frontend/src/lib/components/dbt/DbtModelGraph.svelte +++ b/frontend/src/lib/components/dbt/DbtModelGraph.svelte @@ -25,6 +25,12 @@ DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' import { useDbtRunStatus } from './runStatus.svelte' + import type { DbtGraphPin } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + buildColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' let { workspace, @@ -80,7 +86,19 @@ * buffer rather than a deployed version — as submitted, not as the * editor holds it now. Sent with the selection rather than exposed on * its own so it can never disagree with the SQL the parent shows. */ - buffer: DbtPreviewBuffer | undefined + buffer: DbtPreviewBuffer | undefined, + /** Which graph this node was taken from, so anything else fetched + * about it describes the same project: the editor's own parse job + * when the panel is pinned to one, else the deployed version. Sent + * with the selection for the same reason the buffer is — it must not + * be able to disagree with the node on screen. */ + pin: DbtGraphPin, + /** Column lineage the CONSUMERS of this project declare — a script + * reading a model's column and writing a ducklake one. It comes off + * the same graph response, and the details pane merges it with the + * project's own so a trace crosses that boundary instead of ending + * at it. */ + producerColumns: ColumnLineageGraph ) => void } = $props() @@ -364,6 +382,25 @@ // graph that actually came back. let editorParsed = $derived(refreshJob != undefined && raw?.dbt_snapshot_job === refreshJob) + // Which stored graph is on screen. Anything the details pane fetches about a + // selected node asks for this one, so it cannot describe a node parsed from + // the buffer with the deployed version's answer. + let pin = $derived( + editorParsed && refreshJob ? { jobId: refreshJob } : { scriptHash: deployedHash } + ) + + // What the scripts around this project declare about its columns. Empty for + // the ordinary project nothing downstream annotates. + // + // A consumer is anchored here only by its `// materialize` target: this graph + // is fetched `asset_kinds=dbt` so the canvas is the project and nothing else, + // and `buildColumnGraph`'s other anchor is a ducklake WRITE EDGE, which that + // filter drops. So a script consuming a model and writing a ducklake table it + // never declared contributes no hop in this editor, while it does on the + // pipeline page, whose graph spans both kinds. Widening the request would put + // ducklake nodes on the dbt canvas, which is the opposite of what it is for. + let producerColumns = $derived(graph ? buildColumnGraph(graph) : EMPTY_COLUMN_GRAPH) + // `untrack`, because the effect that reloads the graph clears the selection // through here: reading the graph to describe a selection would subscribe that // effect to the very state its own fetch writes, and it would reload forever. @@ -374,7 +411,9 @@ sel?.kind === 'asset' ? graph?.assets.find((a) => a.kind === sel.asset_kind && a.path === sel.path)?.dbt : undefined, - editorParsed ? parsedBuffer : undefined + editorParsed ? parsedBuffer : undefined, + pin, + producerColumns ) ) } @@ -405,7 +444,6 @@ if (deployedHash != undefined) return 'as of last deploy' return 'never parsed' }) -
@@ -435,8 +473,8 @@ {#if refreshPending}
- Still parsing. A cold worker provisions the dbt engine before it starts; a project - pinned to a worker tag nothing serves waits here indefinitely. + Still parsing. A cold worker provisions the dbt engine before it starts; a project pinned to a + worker tag nothing serves waits here indefinitely. 0) { + const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) + const many = fields.length > 1 + return `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.` + } + // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: + // the worker needs a provider to call and rejects a tool whose name it cannot pass to the + // model. Deploying one would break every flow linking it, so it is refused here. + const blocked = agentConfigRunError(state.args) + if (blocked) { + return blocked + } + // Renaming is not the agent editor's to do: moving the resource leaves every step that links to + // it naming a path that no longer exists, and reconciling those is a feature of its own. A + // renamed path can still reach here, the generic editor writing the same draft row and offering + // a path field, so refuse it rather than performing half of a rename. + if (currentPath && state.path !== currentPath) { + return `This draft renames the agent to ${state.path}. Deploy it from the resource editor instead.` + } + // Only a draft naming another type: the load refuses a resource that is not an agent, while a + // draft the generic resource editor wrote names no type at all and inherits the loaded one. + if (state.resource_type && state.resource_type !== 'ai_agent') { + return `This draft is a ${state.resource_type} resource, not an agent.` + } + return undefined +} + +/** + * Write an agent to its resource from the state the editor holds, which can be ahead of the + * persisted draft row: the form stays editable while a deploy is in flight. Surfaces that deploy + * the row itself go through `deployDraft` instead. + * + * `notAnAgent` separates the one failure that invalidates the caller's whole view of the path, its + * holding something else now, from a write that merely failed. + */ +type AgentWriteResult = { ok: true } | { ok: false; error: string; notAnAgent?: true } + +async function writeAgentResource( + workspace: string, + state: AgentResourceState, + noDeployed: boolean +): Promise { + const body = { + path: state.path, + value: state.args, + description: state.description, + labels: state.labels, + ws_specific: state.wsSpecific + } + try { + if (noDeployed) { + // A create needs a type, and every caller proved this path is an agent before offering it. + await ResourceService.createResource({ + workspace, + requestBody: { ...body, resource_type: state.resource_type ?? 'ai_agent' } + }) + } else { + // The type the caller proved is as old as its own load, and an update carries no type of + // its own: were the path deleted and recreated as something else meanwhile, this write + // would put an agent config inside that resource. Reading it again narrows the window to + // the request rather than to however long the editor or the dialog stayed open. + const current = await ResourceService.getResource({ workspace, path: state.path }) + const refused = agentEditorRefusal(state.path, current.resource_type) + if (refused) { + return { ok: false, error: refused, notAnAgent: true } + } + await ResourceService.updateResource({ workspace, path: state.path, requestBody: body }) + } + } catch (err) { + return { ok: false, error: `Could not save agent: ${err}` } + } + return { ok: true } +} + export interface AgentDraftOptions { /** The `ai_agent` resource being edited. */ path: () => string | undefined @@ -219,6 +309,16 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // config before the autosave lands. state = ((r as any).draft as AgentResourceState | undefined) ?? structuredClone(deployedState) + // Adopt the row's timestamp as this tab's baseline. Without it the first save from + // each tab goes out with no `last_sync`, which the backend treats as unconditional + // and so silently overwrites another tab's newer draft. It also clears any parked + // conflict or failure for the key: a conflict is deliberately sticky (the retry + // keeps the same baseline), and nothing else mounts a resolver for `resource` + // drafts, so re-opening the agent is the only place it can be resolved. + UserDraftDbSyncer.recordRemoteSync( + { workspace: ws, itemKind: 'resource', path }, + (r as { draft_saved_at?: string }).draft_saved_at + ) loading = false await sync.maybeRestore() }, @@ -237,42 +337,9 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { const ws = opts.workspace() const s = state if (!ws || !s) return false - // The editor offers only values, but a transform can arrive from a step that was forked - // before this existed, or from the generic resource editor: say so rather than writing it. - const transformValued = transformValuedBrainKeys(s.args) - if (transformValued.length > 0) { - const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) - const many = fields.length > 1 - sendUserToast( - `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.`, - true - ) - return false - } - // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: - // the worker needs a provider to call and rejects a tool whose name it cannot pass to the - // model. Deploying one would break every flow linking it, so it is refused here. - const blocked = agentConfigRunError(s.args) - if (blocked) { - sendUserToast(blocked, true) - return false - } - // Renaming is not this editor's to do: moving the resource leaves every step that links to it - // naming a path that no longer exists, and reconciling those is a feature of its own. A - // renamed path can still reach here, the generic editor writing the same draft row and - // offering a path field, so refuse it rather than performing half of a rename. - const currentPath = opts.path() - if (currentPath && s.path !== currentPath) { - sendUserToast( - `This draft renames the agent to ${s.path}. Deploy it from the resource editor instead.`, - true - ) - return false - } - // Only a draft naming another type: the load refuses a resource that is not an agent, while a - // draft the generic resource editor wrote names no type at all and inherits the loaded one. - if (s.resource_type && s.resource_type !== 'ai_agent') { - sendUserToast(`This draft is a ${s.resource_type} resource, not an agent.`, true) + const refused = agentDraftDeployRefusal(s, opts.path()) + if (refused) { + sendUserToast(refused, true) return false } // The form stays editable while the request is in flight, so everything below works from a @@ -280,39 +347,15 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // made during the request as saved, and the banner would clear on a value the server never // received; against the snapshot it stays a draft, which is what it is. const submitted = structuredClone($state.snapshot(s)) as AgentResourceState - const body = { - path: submitted.path, - value: submitted.args, - description: submitted.description, - labels: submitted.labels, - ws_specific: submitted.wsSpecific - } - try { - if (noDeployed) { - await ResourceService.createResource({ - workspace: ws, - // A create needs a type, and the load proved this path is an agent before opening. - requestBody: { ...body, resource_type: submitted.resource_type ?? 'ai_agent' } - }) + const written = await writeAgentResource(ws, submitted, noDeployed) + if (!written.ok) { + // A path that is no longer an agent tears this editor down; anything else is a plain error + // the user can retry from the form as it stands. + if (written.notAnAgent) { + refuse(written.error) } else { - // The type this editor proved is as old as the load, and an update carries no type of - // its own: were the path deleted and recreated as something else meanwhile, this write - // would put an agent config inside that resource. Reading it again narrows the window - // to the request rather than to however long the editor stayed open. - const current = await ResourceService.getResource({ workspace: ws, path: submitted.path }) - const refused = agentEditorRefusal(submitted.path, current.resource_type) - if (refused) { - refuse(refused) - return false - } - await ResourceService.updateResource({ - workspace: ws, - path: submitted.path, - requestBody: body - }) + sendUserToast(written.error, true) } - } catch (err) { - sendUserToast(`Could not save agent: ${err}`, true) return false } // The counter the step card's write-back used to report, from the surface that now owns the diff --git a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts index 83660212b3..025a426d28 100644 --- a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts +++ b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts @@ -6,6 +6,8 @@ * resources page. Module-level rather than a context value because what opens it — a step's card, * a list row — unmounts the moment the selection moves. */ +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + export interface AgentEditorTarget { path: string /** The workspace the opener operates on; the nav workspace when absent. */ @@ -61,3 +63,25 @@ export function markAgentWritten(workspace: string | undefined, path: string) { export function agentWriteCount(workspace: string | undefined, path: string | undefined): number { return agentWrites[writeKey(workspace, path)] ?? 0 } + +/** How many times each agent's DRAFT has been saved, for the surfaces that display an agent by + * fetching it. A draft write moves no deployed version, so `agentWriteCount` never sees it, and a + * card keyed on that alone would keep describing the config a test no longer runs. */ +let agentDraftSaves = $state>({}) + +/** Every writer in this document goes through the draft syncer — this editor, the generic resource + * editor — so one subscription answers for them all, firing when the write lands rather than on + * each keystroke. `resource` is the item kind agent draft rows use. In-memory: a save in another + * tab never arrives here, so a card lags it until reload, while what a test runs is read live. */ +UserDraftDbSyncer.onAnySaved(({ workspace, itemKind, path }) => { + if (itemKind !== 'resource') return + const key = writeKey(workspace, path) + agentDraftSaves[key] = (agentDraftSaves[key] ?? 0) + 1 +}) + +export function agentDraftSaveCount( + workspace: string | undefined, + path: string | undefined +): number { + return agentDraftSaves[writeKey(workspace, path)] ?? 0 +} diff --git a/frontend/src/lib/components/flows/agentTelemetry.ts b/frontend/src/lib/components/flows/agentTelemetry.ts index fcfad440ff..87fcb0dad4 100644 --- a/frontend/src/lib/components/flows/agentTelemetry.ts +++ b/frontend/src/lib/components/flows/agentTelemetry.ts @@ -13,6 +13,10 @@ export type ReusableAgentEvent = | 'linked' /** A linked step was forked back into a standalone agent. */ | 'unlinked' + /** A linked agent's unsaved draft was deployed alongside the flow that uses it. */ + | 'draft_deployed_with_flow' + /** A linked agent's unsaved draft was left as a draft when its flow was deployed. */ + | 'draft_kept_on_deploy' export function logReusableAgentUsage(event: ReusableAgentEvent): void { logFeatureUsage('ai_agent', 'reusable', { key: event }) diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 79db0cedb3..a7c867a6f4 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -29,6 +29,7 @@ inputTransformsToAgentConfig, type AIAgentConfig } from '../agentResourceUtils' + import { agentArgsToTransforms } from '../linkedAgentDrafts' import { AGENT_TOOLS_ROW } from '../agentFormFields' import { toolDisplayName, type AgentTool } from '../agentToolUtils' import { useAgentDraft } from '../agentDraft.svelte' @@ -223,19 +224,6 @@ }) }) - /** Every argument the resource carries, as a static transform. `tools` is the roster rather than - * a field, so it rides on the module's own key instead. Not only the keys the form renders: a - * run reads them all, and an agent holding its own `user_message` answers with it when nothing - * overrides it, so a test here has to run the configuration a linked step would. */ - function agentArgsToTransforms(args: AIAgentConfig): Record { - const it: Record = {} - for (const [key, value] of Object.entries(args ?? {})) { - if (key === 'tools' || value === undefined) continue - it[key] = { type: 'static', value } as InputTransform - } - return it - } - /** Everything the form does not model. `inputTransformsToAgentConfig` rebuilds the value from * `AGENT_BRAIN_KEYS` alone, so a key this editor never renders — one a newer backend added, or * the `user_message` default a resource may carry, which the runtime does read when the step diff --git a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte index 3ded2deb9f..22ac424144 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte @@ -183,7 +183,9 @@ const moduleIds = new Set(linkedModulesForAgent(scope, path)) moduleIds.add(at.host.moduleId) return Promise.all( - [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId)) + // With the draft: a deploy leaves none, but a version restore leaves the draft standing and + // it is still what a test of the host step would run. + [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId, true)) ) } diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 3a870f911d..96548b10af 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -4,7 +4,7 @@ import Badge from '$lib/components/common/badge/Badge.svelte' import Path from '$lib/components/Path.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { ResourceService, type InputTransform } from '$lib/gen' + import { ResourceService, type InputTransform, type Resource } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { Bot, ChevronDown, ChevronUp, Save, Unlink, Pencil } from 'lucide-svelte' @@ -19,14 +19,24 @@ type AIAgentConfig, type AgentTool } from '../agentResourceUtils' - import { agentWriteCount, markAgentWritten, openAgentEditor } from '../agentEditorStore.svelte' + import { + agentDraftSaveCount, + agentWriteCount, + markAgentWritten, + openAgentEditor + } from '../agentEditorStore.svelte' import { setLinkedAgentTools, clearLinkedAgentTools, + linkedModulesForAgent, linkedToolsScope } from '../linkedAgentToolsStore.svelte' import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' + import { AgentDraftUnavailable, fetchAgentWithDraft } from '../linkedAgentDrafts' + import type { AgentResourceState } from '../agentDraft.svelte' + import { getLocalDraftHint } from '$lib/localDraftHints.svelte' + import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' import type { AgentTool as AgentToolStrict } from '../agentToolUtils' import { resource } from 'runed' import { untrack } from 'svelte' @@ -65,6 +75,10 @@ // deploy from the agent editor mounted alongside it. Both reads below key on it, so neither // keeps naming the config and version a write has just replaced. let writes = $derived(agentWriteCount(ws, agent)) + // Draft saves as well, for the link fetch: the card shows what a test of this step would run, + // and that is the draft. Only the deploy moves `writes`, so without this the card would keep + // describing the config the agent held before it was edited. + let draftSaves = $derived(agentDraftSaveCount(ws, agent)) let saveDrawer: Drawer | undefined = $state() let newPath = $state('') @@ -75,14 +89,18 @@ type LinkedInfo = { // What this result was fetched for. runed's resource neither aborts nor tags a superseded // request, so a slow fetch can land after a newer one: every consumer gates on these matching - // the current (ws, agent, writes). `writes` is what covers a refetch of the *same* link after - // a deploy — without it a pre-deploy response is indistinguishable from the current one, and - // accepting it republishes the tools the deploy just replaced. + // the current (ws, agent, writes, draftSaves). `writes` is what covers a refetch of the *same* + // link after a deploy — without it a pre-deploy response is indistinguishable from the current + // one, and accepting it republishes the tools the deploy just replaced. `draftSaves` does the + // same for a draft save, which the card follows just as closely. ws?: string path?: string writes: number + draftSaves: number config: AIAgentConfig tools: AgentTool[] + /** The config shown came from the agent's unsaved draft rather than the deployed resource. */ + fromDraft: boolean providerPath?: string providerOk: boolean } @@ -90,14 +108,37 @@ // A linked agent is rigid and read-only: its brain and tools come from the resource. We // load them here for display, and probe the provider resource so we can warn when it isn't // accessible in this workspace (the user then needs to unlink/fork or gain access). + // The draft when there is one, since that is what a test of this step runs. let linkedResource = resource( - () => ({ ws, path: agent, writes }), - async ({ ws, path, writes }): Promise => { + () => ({ ws, path: agent, writes, draftSaves }), + async ({ ws, path, writes, draftSaves }): Promise => { if (!ws || !path) { - return { ws, path, writes, config: {}, tools: [], providerOk: true } + return { + ws, + path, + writes, + draftSaves, + config: {}, + tools: [], + fromDraft: false, + providerOk: true + } + } + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, ws)) + } catch (err) { + // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed + // agent rather than rendering one with no brain and no tools, which reads as "the agent + // is empty" while the Draft badge still says it has unsaved changes. Same fallback the + // graph's tool nodes take; the paths that run or deploy the draft still refuse. + if (!(err instanceof AgentDraftUnavailable)) throw err + response = await ResourceService.getResource({ workspace: ws, path }) + } + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig & { + provider?: { resource?: string } } - const res = await ResourceService.getResource({ workspace: ws, path }) - const cfg = (res.value ?? {}) as AIAgentConfig & { provider?: { resource?: string } } const tools = (cfg.tools ?? []) as AgentTool[] const providerRef = cfg.provider?.resource const providerPath = @@ -116,8 +157,10 @@ ws, path, writes, + draftSaves, config: cfg, tools, + fromDraft: draft != undefined, providerPath, providerOk } @@ -129,7 +172,13 @@ let loadedInfo = $state(undefined) $effect(() => { const current = linkedResource.current - if (current && current.ws === ws && current.path === agent && current.writes === writes) { + if ( + current && + current.ws === ws && + current.path === agent && + current.writes === writes && + current.draftSaves === draftSaves + ) { loadedInfo = current } }) @@ -140,6 +189,12 @@ let brainParams = $derived(summarizeAgentBrain(linkedInfo?.config)) let providerPath = $derived(linkedInfo?.providerPath) let providerOk = $derived(linkedInfo?.providerOk ?? true) + // The hint flips on the first keystroke in the agent editor, so the badge does not wait for the + // debounced autosave and the refetch behind it; the fetched answer covers a draft written + // elsewhere, which no editor here has published an opinion about. + let hasDraft = $derived( + getLocalDraftHint(ws, 'resource', agent ?? '') ?? linkedInfo?.fromDraft ?? false + ) /** The agent the card is about: the one this step links to, or the one being edited. */ let cardPath = $derived(agent) // The version eval runs are recorded against. The resource does not hold it; its newest history @@ -190,9 +245,18 @@ } const loaded = linkedInfo if (loaded) { - claimLinkedToolsFetch(toolScope, moduleId) - // linkedResource types tools loosely; they are the same resource tools the store holds. - setLinkedAgentTools(toolScope, moduleId, loaded.tools as AgentToolStrict[], agent) + // Every step of this flow linking this agent, not just this one. Tools belong to the agent, + // so the sibling steps show the same set, and only the selected step mounts this card: + // without them a draft saved from here leaves their nodes on what the flow load resolved, + // while a test of those steps runs the draft. Claimed like this card's own publish, so a + // sibling's in-flight fetch cannot land afterwards and put the old tools back. + const modules = new Set(linkedModulesForAgent(toolScope, agent)) + modules.add(moduleId) + for (const id of modules) { + claimLinkedToolsFetch(toolScope, id) + // linkedResource types tools loosely; they are the same resource tools the store holds. + setLinkedAgentTools(toolScope, id, loaded.tools as AgentToolStrict[], agent) + } publishedFor = agent } else if (publishedFor !== undefined && publishedFor !== agent) { // The link moved and the new agent hasn't resolved, so the stored tools are the old one's. @@ -358,13 +422,15 @@ // `tools` is one array per module value, so it identifies the step itself — the path alone // would not, since a replacement can carry the same link. const stepMarker = tools - const res = await ResourceService.getResource({ workspace: ws, path }) + // The draft, like the card above and like a test of this step: forking the deployed value + // while the card displays a drafted prompt would hand back something the user never saw. + const { response, draft } = await fetchAgentWithDraft(path, ws) // The module may have been replaced while the fetch was in flight (undo, session drafts); // applying a stale fork would overwrite the restored state. if (agent !== path || tools !== stepMarker) { return false } - const cfg = (res.value ?? {}) as AIAgentConfig + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig // Preserve the flow-local inputs already wired in the step. const local: Record = {} for (const key of AGENT_FLOW_LOCAL_KEYS) { @@ -448,6 +514,15 @@ v{version} {/if} + {#if hasDraft} + + Draft + {#snippet text()} + This agent has unsaved changes. Testing this flow runs the draft, and deploying the + flow offers to deploy it. + {/snippet} + + {/if}
{#if brainParams.length > 0 || inheritedTools.length > 0} diff --git a/frontend/src/lib/components/flows/flowState.ts b/frontend/src/lib/components/flows/flowState.ts index 3123396a9d..595227f57d 100644 --- a/frontend/src/lib/components/flows/flowState.ts +++ b/frontend/src/lib/components/flows/flowState.ts @@ -5,6 +5,7 @@ import { get } from 'svelte/store' import { workspaceStore } from '$lib/stores' import { isFlowModuleTool, agentToolToFlowModule, type AgentTool } from './agentToolUtils' import { linkedToolsScope, setLinkedAgentTools } from './linkedAgentToolsStore.svelte' +import { fetchAgentWithDraft, normalizeAgentRef } from './linkedAgentDrafts' import { loadFlowModuleState } from './flowStateUtils.svelte' import { emptyFlowModuleState } from './utils.svelte' import type { StateStore } from '$lib/utils' @@ -90,7 +91,9 @@ async function mapFlowModule( // the graph can render its tool nodes. They are display-only (their inputs are edited in // the step panel, which infers schemas itself), so no per-tool module state is loaded — // resource tool ids are not flow-unique and must not key into the flow state. - await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id) + // Drafts included: every caller of `initFlowState` is a flow editor, where the graph has + // to show the tools a test would run. Read-only viewers publish for themselves. + await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id, true) } else { // Shape-checked because `tools` is JSON-authored: throwing here would skip the agent's // own state below, leaving it with no schema rather than with no tool schemas. @@ -119,11 +122,17 @@ export async function publishLinkedAgentTools( agentRef: string, workspace: string | undefined, scope: string, - moduleId: string + moduleId: string, + /** Resolve from the agent's unsaved draft when there is one. Editors pass true so the graph + * shows the tool set a test would run; read-only viewers pass false, since a run they are + * displaying used the deployed agent. Required rather than defaulted: an editor call site that + * forgets it republishes the deployed tools over the drafted ones, which reads as the graph + * spontaneously reverting. */ + withDraft: boolean ) { const genKey = `${scope}:${moduleId}` const gen = claimLinkedToolsFetch(scope, moduleId) - const tools = await resolveLinkedAgentTools(agentRef, workspace) + const tools = await resolveLinkedAgentTools(agentRef, workspace, withDraft) if (linkedToolFetchGen.get(genKey) === gen) { setLinkedAgentTools(scope, moduleId, tools, agentRef) } @@ -155,12 +164,27 @@ export function claimLinkedToolsFetch(scope: string, moduleId: string): number { // resource is missing or inaccessible so a broken link never stalls the flow load. export async function resolveLinkedAgentTools( agentRef: string, - workspace?: string + workspace: string | undefined, + withDraft: boolean ): Promise { const ws = workspace ?? get(workspaceStore) if (!ws) return [] - const path = agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') + const path = normalizeAgentRef(agentRef) try { + if (withDraft) { + try { + const { response, draft } = await fetchAgentWithDraft(path, ws) + const value = (draft?.args ?? response.value) as { tools?: AgentTool[] } | undefined + return (value?.tools ?? []) as AgentTool[] + } catch { + // The draft read failed for any reason. This is a display, not a run, so fall through to + // the deployed tools rather than showing an agent with none: an empty node list reads as + // "the agent lost its tools" instead of "we could not reach the server". The paths that + // act on a draft — the previews and the deploy dialog — surface the failure instead. + // Not rethrowing anything here: the outer catch turns every throw into `[]`, so a + // rethrow would skip the very fallback this exists for. + } + } const res = await ResourceService.getResource({ workspace: ws, path }) return ((res.value as { tools?: AgentTool[] } | undefined)?.tools ?? []) as AgentTool[] } catch { diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts new file mode 100644 index 0000000000..563007615f --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -0,0 +1,148 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + inlineAgentDraft, + inlineAgentDrafts, + loadLinkedAgentDrafts, + type LinkedAgentDraft +} from './linkedAgentDrafts' +import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' + +type AiAgentValue = Extract + +function linkedStep(input_transforms: Record): AiAgentValue { + return { + type: 'aiagent', + agent: 'f/team/support', + tool_inputs: { t1: { query: { type: 'javascript', expr: 'flow_input.q' } } }, + input_transforms + } as unknown as AiAgentValue +} + +describe('inlineAgentDraft', () => { + // The overlay order is the worker's: the resource brain first, the step's flow-local inputs on + // top. Reversing it would run the agent author's own `user_message` instead of the flow's. + it('keeps the step wired to the flow while the brain comes from the draft', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }), + { + provider: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' }, + system_prompt: 'answer in french', + user_message: 'the default the agent carries', + tools: [{ id: 't1', summary: 'search' }] + } as any + ) + + expect(inlined.agent).toBeUndefined() + expect(inlined.tools).toEqual([{ id: 't1', summary: 'search' }]) + expect(inlined.input_transforms).toEqual({ + provider: { + type: 'static', + value: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' } + }, + system_prompt: { type: 'static', value: 'answer in french' }, + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }) + // Host bindings are the step's, not the agent's, and the worker overlays them either way. + expect(inlined.tool_inputs).toEqual({ + t1: { query: { type: 'javascript', expr: 'flow_input.q' } } + }) + }) + + // A linked step carries only the flow-local inputs, but one persisted before linking existed can + // still hold stale brain transforms. They must not shadow the draft the test is meant to run. + it('drops brain transforms the step still carries', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'static', value: 'hi' }, + system_prompt: { type: 'static', value: 'stale' } + }), + { system_prompt: 'from the draft' } as any + ) + + expect(inlined.input_transforms).toEqual({ + system_prompt: { type: 'static', value: 'from the draft' }, + user_message: { type: 'static', value: 'hi' } + }) + }) +}) + +describe('inlineAgentDrafts', () => { + // The index is keyed on the bare path while a step may name its agent `$res:`-prefixed, and the + // walk has to reach inside branches and loops. Miss either and every preview silently runs the + // deployed agent — the failure this whole path exists to prevent, and a silent one. + it('reaches a $res:-prefixed link nested in a branch', () => { + const value = { + modules: [ + { + id: 'b', + value: { + type: 'branchone', + default: [], + branches: [ + { + modules: [ + { + id: 'inner', + value: { + type: 'aiagent', + agent: '$res:f/team/support', + tools: [], + input_transforms: { user_message: { type: 'static', value: 'hi' } } + } + } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + const drafts = new Map([ + ['f/team/support', { args: { system_prompt: 'drafted' } } as unknown as LinkedAgentDraft] + ]) + + const inner = (inlineAgentDrafts(value, drafts).modules[0].value as any).branches[0].modules[0] + expect(inner.value.agent).toBeUndefined() + expect(inner.value.input_transforms.system_prompt).toEqual({ + type: 'static', + value: 'drafted' + }) + // The input the flow supplies survives the rewrite. + expect(inner.value.input_transforms.user_message).toEqual({ type: 'static', value: 'hi' }) + }) +}) + +// A link the user cannot resolve is an ordinary state and must not block the flow; anything else is +// an outage, and answering "no draft" to one would silently test or deploy against the deployed +// agent while the editor shows the draft. +describe('loadLinkedAgentDrafts error handling', () => { + function failWith(status: number | undefined) { + return async () => { + const err: Error & { status?: number } = new Error('boom') + err.status = status + throw err + } + } + + beforeEach(() => { + vi.restoreAllMocks() + }) + + it.each([401, 403, 404])('treats %i as no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).resolves.toEqual(new Map()) + }) + + it.each([500, undefined])('propagates %s rather than reporting no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).rejects.toThrow( + 'Could not load the agent f/team/support' + ) + }) +}) diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts new file mode 100644 index 0000000000..8da732e7ff --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -0,0 +1,229 @@ +import { + ResourceService, + type FlowModule, + type FlowValue, + type InputTransform, + type Resource +} from '$lib/gen' +import { UserDraft } from '$lib/userDraft.svelte' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' +import { dfs } from './dfs' +import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils' +import type { AgentResourceState } from './agentDraft.svelte' +import type { AgentTool } from './agentToolUtils' + +/** A step names its agent bare or as `$res:`/`res://`; all three are the same agent, + * and a draft index has to answer for a lookup written any of those ways. Same normalization as + * `linkedAgentToolsStore`, and as the `trim_start_matches` the worker applies. */ +export function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + +/** Every `ai_agent` resource this flow links to, deduped. `dfs` walks agent tool nodes as well as + * branches and loops, so a nested linked agent tool is included. */ +export function linkedAgentPaths(value: FlowValue | undefined): string[] { + if (!value?.modules) return [] + const paths = new Set() + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent) { + paths.add(normalizeAgentRef(v.agent)) + } + } + return [...paths] +} + +/** + * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then + * what a `get_draft` response carried. + * + * Only the live cell is reliably current, and only while an editor holds it: `releaseEntry` drops + * the cached write at refcount 0 on purpose, so once the agent editor closes the persisted row is + * the sole answer. Read through `fetchAgentWithDraft`, which settles that row first. + */ +export function agentDraftState( + response: { draft?: unknown }, + path: string, + workspace: string | undefined +): AgentResourceState | undefined { + const live = UserDraft.get('resource', path, { workspace }) + return live ?? (response.draft as AgentResourceState | undefined) +} + +/** A refusal that already names the agent and says what is wrong with it, so a caller wrapping it + * would only repeat itself. */ +export class AgentDraftUnavailable extends Error {} + +/** + * An agent's resource together with the draft a run of it would use. + * + * The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and + * closing the agent editor releases the in-memory cell without cancelling that pending POST — so + * testing or deploying right after closing would otherwise read a row the last edits have not + * reached yet. `flush` replays the parked save and is a no-op when there is none. + */ +export async function fetchAgentWithDraft( + path: string, + workspace: string +): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + const query = { workspace, itemKind: 'resource' as const, path } + await UserDraftDbSyncer.flush(query) + // `flush` resolves whether or not the save actually landed: `postSave` catches network and + // server errors into its failure map, and answers a conflicting write by parking a snapshot, + // returning normally in both cases. The row about to be read is then older than the edit still + // held in the browser, and nothing downstream could tell. Running that row is a test of the + // wrong agent; deploying it is worse, because the deploy deletes the draft and takes the newer + // edit with it. Neither is recoverable from here, so refuse the read. + const failure = UserDraftDbSyncer.getState(query).failureMessage + if (failure) { + throw new AgentDraftUnavailable(`The unsaved changes to ${path} could not be saved: ${failure}`) + } + if (UserDraftDbSyncer.getConflict(query).conflict) { + throw new AgentDraftUnavailable( + `The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.` + ) + } + const response = await ResourceService.getResource({ workspace, path, getDraft: true }) + return { response, draft: agentDraftState(response, path, workspace) } +} + +/** One linked agent whose resource the user has an unsaved draft for. */ +export interface LinkedAgentDraft { + path: string + /** The draft's resource value: what a run of this agent would use. */ + args: AIAgentConfig + /** The whole draft row, as the resource editors write it — the deploy payload. */ + state: AgentResourceState + /** No deployed row at this path, so deploying has to create rather than update. */ + noDeployed: boolean + /** Of the deployed resource, for `agentDraftCanWrite`. */ + extraPerms: Record +} + +/** Whether `user` may write this agent's resource. Split from the load so that resolving the + * drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it + * looks the user up once for every agent it lists. */ +export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean { + return canWrite(draft.path, draft.extraPerms, user) +} + +/** A link that cannot resolve for the user rather than because something went wrong: the agent was + * deleted, or sits in a folder they cannot read. Both are ordinary states of a rigid link, and + * neither should stop the caller — the flow still tests and deploys, against the deployed agent. + * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy + * the wrong configuration, which is the whole thing this module exists to prevent. */ +function isExpectedLinkFailure(err: unknown): boolean { + const status = (err as { status?: number } | null | undefined)?.status + return status === 401 || status === 403 || status === 404 +} + +/** + * The unsaved draft of every given `ai_agent` path, for the paths that have one. + * + * Throws when a path fails to load for any reason other than being missing or unreadable, so a + * caller cannot mistake an outage for an agent with nothing unsaved. + */ +export async function loadLinkedAgentDrafts( + paths: string[], + workspace: string | undefined +): Promise> { + const out = new Map() + if (!workspace || paths.length === 0) return out + await Promise.all( + paths.map(async (path) => { + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, workspace)) + } catch (err) { + if (isExpectedLinkFailure(err)) return + if (err instanceof AgentDraftUnavailable) throw err + throw new Error(`Could not load the agent ${path}: ${err}`) + } + if (!draft) return + out.set(path, { + path, + args: (draft.args ?? {}) as AIAgentConfig, + state: draft, + noDeployed: Boolean((response as { no_deployed?: boolean }).no_deployed), + extraPerms: response.extra_perms ?? {} + }) + }) + ) + return out +} + +/** + * Every argument a saved agent carries, as a static input transform. Not only the keys the agent + * form renders: a run reads them all, and an agent holding its own `user_message` answers with it + * when nothing overrides it. `tools` is the step's own roster rather than an input, so it rides on + * the module's `tools` key instead. + */ +export function agentArgsToTransforms(args: AIAgentConfig): Record { + const it: Record = {} + for (const [key, value] of Object.entries(args ?? {})) { + if (key === 'tools' || value === undefined) continue + it[key] = { type: 'static', value } as InputTransform + } + return it +} + +type AiAgentValue = Extract + +/** + * The standalone step a linked step's draft would run as: the draft's brain and tools inlined, with + * the step's own flow-local inputs kept on top. + * + * The overlay order is the worker's (`ai_executor.rs`): its linked branch interpolates the whole + * resource brain and only then writes `user_message`/`user_attachments` back from the step's own + * args. `tool_inputs` stays untouched — the worker overlays it onto the tools in both branches, so + * an inlined step keeps the host flow's tool bindings. + */ +export function inlineAgentDraft(value: AiAgentValue, args: AIAgentConfig): AiAgentValue { + const { agent: _agent, ...rest } = value + return { + ...rest, + tools: (args.tools ?? []) as AgentTool[], + input_transforms: { + ...agentArgsToTransforms(args), + ...flowLocalInputs(value.input_transforms as Record) + } + } as AiAgentValue +} + +/** + * Replace every linked agent step that has a draft with the draft's own configuration, so a preview + * runs what the agent editor is showing rather than the deployed resource. Returns a new value: the + * flow editor hands its live store object to previews. + */ +export function inlineAgentDrafts( + value: FlowValue, + drafts: Map +): FlowValue { + if (drafts.size === 0) return value + // JSON rather than `structuredClone`: the flow editor's value is a Svelte `$state` proxy, which + // `structuredClone` refuses outright. A flow value is JSON by definition — it is about to be + // posted as one — so the round trip loses nothing this preview would have carried. + const next = JSON.parse(JSON.stringify(value)) as FlowValue + for (const module of dfs(next.modules ?? [], (m) => m)) { + const v = module?.value as AiAgentValue | undefined + if (v?.type !== 'aiagent' || !v.agent) continue + const draft = drafts.get(normalizeAgentRef(v.agent)) + if (!draft) continue + module.value = inlineAgentDraft(v, draft.args) + } + return next +} + +/** Load the drafts this flow's linked agents have and inline them. The whole substitution, for a + * caller holding nothing but the value it is about to preview. */ +export async function withAgentDrafts( + value: FlowValue, + workspace: string | undefined +): Promise { + const paths = linkedAgentPaths(value) + if (paths.length === 0) return value + return inlineAgentDrafts(value, await loadLinkedAgentDrafts(paths, workspace)) +} diff --git a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts index ea1674e0b6..29207f58f7 100644 --- a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts +++ b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts @@ -8,7 +8,9 @@ vi.mock('./agentToolUtils', () => ({ isFlowModuleTool: () => false, agentToolToFlowModule: (t: unknown) => t })) -vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } })) +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } +})) import { claimLinkedToolsFetch, @@ -38,8 +40,8 @@ describe('linked tools fetch guard', () => { ) .mockResolvedValueOnce({ value: { tools: [tool('new')] } } as never) - const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') - await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step') + const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) + await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step', false) release?.({ value: { tools: [tool('old')] } }) await stale @@ -56,7 +58,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) setLinkedAgentTools(scope, 'step', [tool('kept')], 'u/admin/a') invalidateLinkedToolsFetches(scope) @@ -73,7 +75,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) claimLinkedToolsFetch(scope, 'step') setLinkedAgentTools(scope, 'step', [tool('direct')], 'u/admin/a') release?.({ value: { tools: [tool('stale')] } }) diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index 207e8612e9..d84228dd35 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -8,8 +8,13 @@ import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { disableHubStore } from '$lib/stores' + import { disableHubStore, workspaceStore } from '$lib/stores' import { logHubScriptPick } from '$lib/utils/featureUsage' + import { + alphabetical, + byPopularity, + localCountsByIntegration + } from '$lib/components/pickerPopularity' interface Props { kind?: HubScriptKind & string @@ -46,19 +51,25 @@ }[] = $state([]) let allApps: string[] = $state([]) + let popularity: (a: string, b: string) => number = $state(alphabetical) let apps: string[] = $derived.by(() => - filter.length > 0 ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() : allApps + filter.length > 0 + ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity) + : allApps ) async function getAllApps(filterKind: typeof kind) { if ($disableHubStore) return try { hubNotAvailable = false - allApps = ( - await IntegrationService.listHubIntegrations({ - kind: filterKind - }) - ).map((x) => x.name) + // Independent reads, so they share one round trip before first paint. + const [integrations, local] = await Promise.all([ + IntegrationService.listHubIntegrations({ kind: filterKind }), + $workspaceStore ? localCountsByIntegration($workspaceStore) : {} + ]) + const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) + popularity = byPopularity(hubPicks, local) + allApps = integrations.map((x) => x.name).sort(popularity) } catch (err) { console.error('Hub is not available') allApps = [] @@ -154,72 +165,73 @@ {#if $disableHubStore} {:else} -
- {@render children?.()} -
- - {#if loading} - - {/if} -
-
- -{#if hubNotAvailable} - - Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. - -{:else if (items.length > 0 && apps.length > 0) || !loading} - - {#if items.length == 0} - - {:else} -
    - {#each items as item (item.path)} -
  • - -
    • - {/each} -
- {/if} - {#if items.length == 20} -
- There are more items than being displayed. Refine your search. +
+ {@render children?.()} +
+ + {#if loading} + + {/if}
+
+ + {#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the + Hub in the instance settings. + + {:else if (items.length > 0 && apps.length > 0) || !loading} + + {#if items.length == 0} + + {:else} +
    + {#each items as item (item.path)} +
  • + +
    • + {/each} +
+ {/if} + {#if items.length == 20} +
+ There are more items than being displayed. Refine your search. +
+ {/if} + {:else} + {#each Array(10).fill(0) as _} + + {/each} {/if} -{:else} - {#each Array(10).fill(0) as _} - - {/each} -{/if} {/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 5204892051..d32e82c702 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -44,12 +44,17 @@ import { Circle, ExternalLink } from 'lucide-svelte' import Popover from '$lib/components/Popover.svelte' import { usePromise } from '$lib/svelte5Utils.svelte' - import { disableHubStore, hubBaseUrlStore, userStore } from '$lib/stores' + import { disableHubStore, hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores' import { get } from 'svelte/store' import Button from '$lib/components/common/button/Button.svelte' import { Alert } from '$lib/components/common' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' import { logHubScriptPick } from '$lib/utils/featureUsage' + import { + alphabetical, + byPopularity, + localCountsByIntegration + } from '$lib/components/pickerPopularity' let customUi: undefined | FlowBuilderWhitelabelCustomUi = getContext('customUi') @@ -94,9 +99,10 @@ }: Props = $props() let allApps: string[] = $state([]) + let popularity: (a: string, b: string) => number = $state(alphabetical) $effect(() => { if (filter.length > 0) { - apps = Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() + apps = Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity) } else { apps = allApps } @@ -106,9 +112,14 @@ if ($disableHubStore) return try { hubNotAvailable = false - allApps = (await listHubIntegrationsCached({ kind: filterKind, refreshCount })).map( - (x) => x.name - ) + // Independent reads, so they share one round trip before first paint. + const [integrations, local] = await Promise.all([ + listHubIntegrationsCached({ kind: filterKind, refreshCount }), + $workspaceStore ? localCountsByIntegration($workspaceStore) : {} + ]) + const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) + popularity = byPopularity(hubPicks, local) + allApps = integrations.map((x) => x.name).sort(popularity) } catch (err) { console.error('Failed to fetch hub integrations:', err) allApps = [] diff --git a/frontend/src/lib/components/flows/utils.svelte.ts b/frontend/src/lib/components/flows/utils.svelte.ts index 8d8e4366e5..df27960dc7 100644 --- a/frontend/src/lib/components/flows/utils.svelte.ts +++ b/frontend/src/lib/components/flows/utils.svelte.ts @@ -17,6 +17,7 @@ import { get } from 'svelte/store' import type { FlowModuleState } from './flowState' import { type PickableProperties, dfs } from './previousResults' import { forEachFlowModule } from './dfs' +import { withAgentDrafts } from './linkedAgentDrafts' import { NEVER_TESTED_THIS_FAR } from './models' import { sendUserToast } from '$lib/toast' import type { ExtendedOpenFlow } from './types' @@ -187,6 +188,12 @@ export function jobsToResults(jobs: Job[]) { }) } +/** + * Run the flow the editor currently holds. A step linked to a saved agent runs that agent's + * unsaved draft when there is one (`withAgentDrafts`), so testing exercises what the agent editor + * is showing rather than the deployed resource — the same rule the agent editor's own test pane + * follows. The value passed in is left alone; only what goes to the server is substituted. + */ export async function runFlowPreview( args: Record, flow: OpenFlow & { tag?: string }, @@ -198,14 +205,15 @@ export async function runFlowPreview( // editor; falls back to the navigation workspace for full-page previews. workspace?: string ) { - const newFlow = flow + const ws = workspace ?? get(workspaceStore) ?? '' + const value = await withAgentDrafts(flow.value, ws) return await JobService.runFlowPreview({ - workspace: workspace ?? get(workspaceStore) ?? '', + workspace: ws, requestBody: { args, - value: newFlow.value, + value, path: path, - tag: newFlow.tag, + tag: flow.tag, restarted_from: restartedFrom, temp_script_refs: tempScriptRefs }, diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte index 8183be16d6..db5abfde20 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte @@ -7,7 +7,7 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import AssetNode from './AssetNode.svelte' import type { FlowGraphAssetContext } from '$lib/components/flows/types' - import { getContext } from 'svelte' + import { getContext, untrack } from 'svelte' import { assetEq } from '$lib/components/assets/lib' import { getNodeColorClasses } from '../../util' @@ -24,15 +24,17 @@ data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx?.val.selectedAsset, asset)) ) - let wasOpenedBecauseOfExternalSelected = false + // Open while a sibling asset node is hovered and one of the hidden assets is the same asset. + let openedByHover = $state(false) $effect(() => { - if (includesSelected && !isOpen) { - isOpen = true - wasOpenedBecauseOfExternalSelected = true - } - if (wasOpenedBecauseOfExternalSelected && !includesSelected) { + if (includesSelected) { + if (!untrack(() => isOpen)) { + isOpen = true + openedByHover = true + } + } else if (untrack(() => openedByHover)) { isOpen = false - wasOpenedBecauseOfExternalSelected = false + openedByHover = false } }) const colors = $derived(getNodeColorClasses(undefined, includesSelected)) @@ -41,9 +43,17 @@ {#snippet children({ darkMode })} + {#snippet trigger()} - - +{data.overflowedAssets.length} - - {/snippet} + +{data.overflowedAssets.length} + {/snippet} {#snippet content()} - -
    - {#each data.overflowedAssets as asset} -
  • - -
    • - {/each} -
- - {/snippet} +
    + {#each data.overflowedAssets as asset} +
  • + +
    • + {/each} +
+ {/snippet} {/snippet} diff --git a/frontend/src/lib/components/meltComponents/Popover.svelte b/frontend/src/lib/components/meltComponents/Popover.svelte index f850e53f6c..6a6b7d681b 100644 --- a/frontend/src/lib/components/meltComponents/Popover.svelte +++ b/frontend/src/lib/components/meltComponents/Popover.svelte @@ -60,6 +60,8 @@ documentationLink?: string | undefined disableFocusTrap?: boolean openFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined + /** Element to focus when the popover closes; defaults to the trigger, `null` leaves focus alone. */ + closeFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined escapeBehavior?: EscapeBehaviorType enableFlyTransition?: boolean onKeyDown?: (e: KeyboardEvent) => void @@ -99,6 +101,7 @@ documentationLink = undefined, disableFocusTrap = false, openFocus = undefined, + closeFocus = undefined, escapeBehavior = 'close', enableFlyTransition = false, onKeyDown = () => {}, @@ -133,6 +136,7 @@ disableFocusTrap: untrack(() => disableFocusTrap), escapeBehavior: untrack(() => escapeBehavior), openFocus: untrack(() => openFocus), + closeFocus: untrack(() => closeFocus), onOpenChange: ({ curr, next }) => { if (curr != next) { dispatch('openChange', next) diff --git a/frontend/src/lib/components/pickerPopularity.test.ts b/frontend/src/lib/components/pickerPopularity.test.ts new file mode 100644 index 0000000000..f9440981ad --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from 'vitest' +import { alphabetical, byPopularity, totalLocalCountsByApp } from './pickerPopularity' + +const order = (names: string[], hub: Record, local: Record = {}) => + [...names].sort(byPopularity(hub, local)) + +describe('byPopularity', () => { + // The tier that stops a filling-up hub from squeezing the workspace's own stack out of + // the ordering: a global pick count grows without bound, a local one does not. + it('leads with what the workspace uses, whatever the hub says', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { stripe: 1 })).toEqual(['stripe', 'slack']) + }) + + it('ranks the used types among themselves by hub picks', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { slack: 1, stripe: 1 })).toEqual([ + 'slack', + 'stripe' + ]) + }) + + it('breaks a hub tie on how much the workspace uses it', () => { + expect(order(['slack', 'stripe'], { slack: 5, stripe: 5 }, { slack: 1, stripe: 2 })).toEqual([ + 'stripe', + 'slack' + ]) + }) + + it('ranks the unused types by hub picks, below every used one', () => { + expect(order(['ably', 'github', 'stripe'], { ably: 900, github: 5 }, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + it('falls back to alphabetical for everything neither signal ranks', () => { + expect(order(['stripe', 'ably', 'github'], { github: 3 })).toEqual(['github', 'ably', 'stripe']) + }) + + it('orders on local usage alone when the hub ranks nothing', () => { + expect(order(['stripe', 'ably', 'github'], {}, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + // The lists render before either signal lands, and one of them arrives in a server-side + // HashMap's iteration order, so the resting comparator has to sort rather than no-op. + it('leaves an alphabetical order with no signal at all', () => { + expect(['stripe', 'ably', 'github'].sort(alphabetical)).toEqual(['ably', 'github', 'stripe']) + }) +}) + +describe('totalLocalCountsByApp', () => { + const HUB = [ + { name: 'discord_webhook', app: 'discord', picks: 0 }, + { name: 'discord_bot_configuration', app: 'discord', picks: 0 }, + { name: 'ms_teams_webhook', app: 'msteams', picks: 0 }, + { name: 'slack', app: 'slack', picks: 0 } + ] + + // The integration pickers list app names, the counts arrive keyed by resource type, and + // the two only usually agree. Without the mapping a workspace whose Discord credential is + // a `discord_webhook` never reaches the used-here tier at all. + it('totals a resource type under the integration it belongs to', () => { + expect(totalLocalCountsByApp({ discord_webhook: 2 }, HUB)).toEqual({ discord: 2 }) + }) + + it('sums the several types one integration can have', () => { + expect( + totalLocalCountsByApp({ discord_webhook: 2, discord_bot_configuration: 1 }, HUB) + ).toEqual({ discord: 3 }) + }) + + // What a workspace-made type, and an unreachable hub, both leave every entry with. + it('keeps a type the hub has no mapping for under its own name', () => { + expect(totalLocalCountsByApp({ c_acme: 1, slack: 2 }, HUB)).toEqual({ c_acme: 1, slack: 2 }) + }) +}) diff --git a/frontend/src/lib/components/pickerPopularity.ts b/frontend/src/lib/components/pickerPopularity.ts new file mode 100644 index 0000000000..2d585ff858 --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.ts @@ -0,0 +1,149 @@ +import { get } from 'svelte/store' +import { ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { isCustomResourceTypeName } from './resourceTypeDisplay' + +/** + * How often something has been picked or used, keyed by integration or resource type name. + * A name the caller lists but this map does not mention counts as zero, which is what an + * unpicked entry and an absent signal both mean. + */ +export type PopularityCounts = Record + +/** + * The signals are read on every picker open, so they are cached briefly; both resolve to an + * empty map rather than rejecting, since an ordering hint is never worth a broken list. + */ +const CACHE_MS = 60_000 + +type HubResourceTypeInfo = { name: string; app: string; picks: number } + +const hubInfoCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + return await ResourceService.listHubResourceTypeInfo({ workspace }) + } catch { + return [] + } + }, + { invalidateMs: CACHE_MS } +) + +const localCountsCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + const counts = await ResourceService.listResourceCountsByType({ workspace }) + return Object.fromEntries(counts.map((c) => [c.resource_type, c.count])) + } catch { + return {} + } + }, + { invalidateMs: CACHE_MS } +) + +/** + * What the hub sees people pick, per resource type. Empty on a hub that counts nothing, + * and on an instance that has switched the hub off — a closed environment must not spend a + * request on hub.windmill.dev just to order a list. + */ +export async function hubResourceTypePicks(workspace: string): Promise { + if (get(disableHubStore)) return {} + const info = await hubInfoCached({ workspace }) + return Object.fromEntries(info.map((rt) => [rt.name, rt.picks])) +} + +/** + * How many resources of each type this workspace holds — the only evidence about this + * particular team. Keyed by resource type, which is what the add-resource drawer lists. + */ +export function localResourceTypeCounts(workspace: string): Promise { + return localCountsCached({ workspace }) +} + +/** + * The same counts totalled per integration, which is what the flow step picker lists. + * + * A type usually shares its integration's name, but often enough it does not: + * `discord_webhook` and `discord_bot_configuration` are both Discord, `ms_teams_webhook` and + * `azure_bot` are both MS Teams. Only the hub knows that, so a workspace whose Discord + * credential is a `discord_webhook` would otherwise read as one that has never touched + * Discord — and since local usage is the leading tier, that decides which half of the list + * the integration lands in, not merely its position within one. + * + * A type the hub has no mapping for counts under its own name, which is the right guess and + * also what an unreachable hub leaves every type with. + */ +export async function localCountsByIntegration(workspace: string): Promise { + const [counts, info] = await Promise.all([ + localCountsCached({ workspace }), + get(disableHubStore) ? Promise.resolve([]) : hubInfoCached({ workspace }) + ]) + return totalLocalCountsByApp(counts, info) +} + +/** The mapping half of {@link localCountsByIntegration}, separated so it can be tested alone. */ +export function totalLocalCountsByApp( + counts: PopularityCounts, + hub: { name: string; app: string }[] +): PopularityCounts { + const appOf = new Map(hub.map((rt) => [rt.name, rt.app])) + const byApp: PopularityCounts = {} + for (const [name, count] of Object.entries(counts)) { + const app = appOf.get(name) ?? name + byApp[app] = (byApp[app] ?? 0) + count + } + return byApp +} + +/** + * Tell the hub a resource type was taken into a workspace, which is what its ranking counts. + * Fire-and-forget: a hub that does not count picks must not be felt by the user who just + * saved a resource. Workspace-made types exist on no hub, so they are not reported. + */ +export function recordHubResourceTypePick(workspace: string, resourceType: string): void { + if (get(disableHubStore)) return + if (!resourceType || isCustomResourceTypeName(resourceType)) return + ResourceService.pickHubResourceType({ workspace, name: resourceType }).catch(() => {}) +} + +/** + * Orders the lists that offer hub content: integrations in the flow step picker, resource + * types in the add-resource drawer. + * + * Four tiers. **Whether this workspace already holds a resource of the type leads**, then the + * hub's pick count, then how many local resources there are, then the name. + * + * Used-here leads rather than merely breaking hub ties because the two counts are on + * incomparable scales: a hub pick count is global and grows without bound, a local count is + * usually single digits. Ranked the other way round, local usage only ever sorts the slice + * where hub counts are equal — which, since they are distinct integers, is just the tail + * that nobody has picked. That reads fine on a hub with few picks and silently stops + * mattering as one fills up, so the ordering would drift away from the workspace's own + * stack with no change to this code. + * + * Within each half the hub decides, so "yours" and "everyone's" are both honoured rather + * than blended with a weighting constant that would need tuning. Alphabetical is the floor, + * and it is where an entry neither signal knows about lands. + */ +export function byPopularity( + hub: PopularityCounts, + local: PopularityCounts +): (a: string, b: string) => number { + const usedHere = (name: string) => ((local[name] ?? 0) > 0 ? 1 : 0) + return (a, b) => + usedHere(b) - usedHere(a) || + (hub[b] ?? 0) - (hub[a] ?? 0) || + (local[b] ?? 0) - (local[a] ?? 0) || + a.localeCompare(b) +} + +/** + * The ordering to hold before either signal has landed: the alphabetical floor, which is + * what `byPopularity` degrades to anyway. + * + * A list has to be sorted by *something* from its first paint — one source of these names + * is a `HashMap` on the server, so leaving them unsorted means hash order, which differs + * between processes. + */ +export const alphabetical = byPopularity({}, {}) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index a117d2769c..084ad9d492 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -427,7 +427,7 @@ path = `/apps/get/${e.path}` break case 'raw_app': - path = `/raw_apps/get/${e.path}` + path = `/apps_raw/get/${e.path}` break default: path = '/' diff --git a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts index ba09167d6d..98d38ac25f 100644 --- a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts +++ b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts @@ -2,9 +2,21 @@ import { untrack } from 'svelte' import { deepEqual } from 'fast-equals' import { UserDraft, normalizeDraftForCompare, type UserDraftItemKind } from '$lib/userDraft.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' +import { onUserInput } from '$lib/userDraftEditGate' type Cfg = Record +/** + * Detach a config from whatever holds it. The draft cell is deeply reactive, + * so handing its object straight to `applyCfg` would make the form's own + * `$state` (a schedule's `args`, say) the very object the cell holds — every + * later keystroke would then mutate the draft in place behind the autosave's + * back. + */ +function snapshotCfg(cfg: V): V { + return structuredClone($state.snapshot(cfg)) as V +} + /** * Whether `a` differs from `b` after `normalizeDraftForCompare` (JSON * round-trip to drop `undefined`-valued keys, plus ignored deploy-directive @@ -84,8 +96,10 @@ export interface TriggerDraftSync { * …)` (another tab, a programmatic write) propagate into the open editor. * * - **apply-effect**: reflects external `handle.draft` changes into the form. + * - **absorb-effect**: folds the form's own settling into the baseline until + * the user's first input, so a schema that moved on is not a draft. * - **persist-effect**: writes form edits back through the handle, dropping - * the draft when the form is back at the deployed baseline. + * the draft when the form is back at the baseline. * * Both effect bodies are `untrack`ed and gated by `cfgDiffers` * idempotence so they can't feed back into each other. Must be called once @@ -99,14 +113,56 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft }) const handle = $derived(handles[0]) + // Gated until the user puts something in (see `onUserInput`): the baseline + // absorbs whatever the form settles on and nothing persists, so an untouched + // trigger never reports unsaved changes. A drawer opened ON a restored draft + // absorbs nothing — that divergence is the user's own. + let settledBaseline: Cfg | undefined = $state(undefined) + let userEdited = $state(false) + let openedOnDraft = $state(false) + // These editors are mounted by the list page, not by the drawer, so input + // arriving while the drawer is still loading is the click that opened it + // (or anything else on the page behind) — never an edit to this form. + onUserInput(() => { + if (!opts.drawerLoading()) userEdited = true + }) + + /** The deployed config, plus whatever the form settled on by itself. */ + const baseline = $derived(settledBaseline ?? opts.deployed()) + + $effect(() => { + // A reload re-opens the gate's window: the drawer is being pointed at a + // different trigger, or the same one re-read from the backend. The click + // that opened it landed before this, hence the reset of `userEdited`. + if (!opts.drawerLoading()) return + untrack(() => { + settledBaseline = undefined + userEdited = false + openedOnDraft = false + }) + }) + + // absorb-effect: pre-edit form drift joins the baseline. + $effect(() => { + if (opts.drawerLoading() || userEdited || openedOnDraft) return + const cfg = opts.getCfg() + const deployed = opts.deployed() + if (cfg == null || deployed == null) return + // Snapshot before untracking: `getCfg` hands back the form's `$state` + // objects by reference, so only a deep read subscribes to the nested + // writes the form makes as it settles. + const settled = snapshotCfg(cfg) + untrack(() => { + if (cfgDiffers(settled, settledBaseline ?? deployed)) settledBaseline = settled + }) + }) + // Live "is there a local draft?" — the form diverges from the deployed // baseline. Gated on `!drawerLoading` (the baseline isn't settled yet // mid-load) and on a non-null baseline (a brand-new trigger has none, so // "unsaved changes" / discard-to-deployed is meaningless there). const hasDraft = $derived( - !opts.drawerLoading() && - opts.deployed() != null && - cfgDiffers(opts.getCfg() as Cfg, opts.deployed() as Cfg) + !opts.drawerLoading() && baseline != null && cfgDiffers(opts.getCfg() as Cfg, baseline as Cfg) ) // Reactive "banner is possible" — depends on `drawerLoading()` so it @@ -131,7 +187,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft if (opts.drawerLoading() || d == null) return untrack(() => { if (cfgDiffers(d, opts.getCfg() as Cfg)) { - void opts.applyCfg(d) + void opts.applyCfg(snapshotCfg(d)) } }) }) @@ -148,30 +204,32 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft discardTimer = undefined if (opts.drawerLoading()) return const cfg = opts.getCfg() - const deployed = opts.deployed() const h = handle if (!h || cfg == null) return - if (!cfgDiffers(cfg, deployed) && cfgDiffers(h.draft, deployed)) { - discard(opts.path(), deployed, true) + if (!cfgDiffers(cfg, baseline) && cfgDiffers(h.draft, baseline)) { + discard(opts.path(), baseline, true) } }, 600) } // persist-effect: form edits → handle; drop the draft when back at the - // deployed baseline. + // baseline. $effect(() => { if (opts.drawerLoading() || !opts.path()) return + // Nothing persists before the user's first input — the form's own + // settling is not an edit, and gating here rather than relying on the + // absorb-effect having run first keeps the two effects order-independent. + if (!userEdited && !openedOnDraft) return const cfg = opts.getCfg() if (cfg == null) return untrack(() => { const h = handle if (!h) return - const deployed = opts.deployed() - if (cfgDiffers(cfg, deployed)) { + if (cfgDiffers(cfg, baseline)) { if (cfgDiffers(cfg, h.draft)) h.draft = cfg - } else if (cfgDiffers(h.draft, deployed)) { - // Only when a draft actually exists to drop: `h.draft` equals - // `deployed` right after a discard or the post-load seed. + } else if (cfgDiffers(h.draft, baseline)) { + // Only when a draft actually exists to drop: `h.draft` equals the + // baseline right after a discard or the post-load seed. scheduleAutoDiscard() } }) @@ -202,7 +260,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft return hasBaseline }, get deployed() { - return opts.deployed() + return baseline }, get current() { return opts.getCfg() @@ -211,8 +269,14 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const d = handle?.draft if (cfgDiffers(d, opts.getCfg() as Cfg)) { // Overlay the local autosave on the just-loaded backend config. - await opts.applyCfg(d) + await opts.applyCfg(snapshotCfg(d)) } + // The form is not rendered while the drawer loads, so anything that + // diverges from the deployed config right now is a draft restored onto + // it — by the overlay above, or by the editor from the backend before + // calling this — never the form settling. Absorbing that into the + // baseline would hide the user's own work behind a clean drawer. + openedOnDraft = cfgDiffers(opts.getCfg() as Cfg, opts.deployed()) // Adopt the post-load form state as the cell's baseline without // POSTing, consuming the entry's one-shot first-write seed guard. // Trigger drawers never write the cell programmatically on open, so @@ -221,14 +285,20 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const p = opts.path() const cfg = opts.getCfg() if (ws && p && cfg != null) { - UserDraft.seed(opts.itemKind, p, structuredClone($state.snapshot(cfg)) as Cfg, { + UserDraft.seed(opts.itemKind, p, snapshotCfg(cfg) as Cfg, { workspace: ws }) } }, async resetToDeployed(path: string) { - const deployedCfg = structuredClone($state.snapshot(opts.deployed())) as Cfg + const deployedCfg = snapshotCfg(opts.deployed()) as Cfg discard(path, deployedCfg) + // Nothing of the user's is left in the form, so the gate closes again + // — otherwise the form settles on the schema's values a second time + // and the discarded draft comes straight back. + settledBaseline = undefined + userEdited = false + openedOnDraft = false await opts.applyCfg(deployedCfg) }, discard diff --git a/frontend/src/lib/scriptEditorSchema.ts b/frontend/src/lib/scriptEditorSchema.ts new file mode 100644 index 0000000000..7bd13b32a7 --- /dev/null +++ b/frontend/src/lib/scriptEditorSchema.ts @@ -0,0 +1,90 @@ +import type { Schema, SupportedLanguage } from '$lib/common' +import type { Script } from '$lib/gen' +import { inferArgs } from '$lib/infer' +import { emptySchema } from '$lib/utils' +import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + +/** + * A `// partitioned` pipeline script is materialized one slice at a time and + * receives the slice as a runtime `partition` arg (the cascade injects it in + * production). It isn't a code parameter, so schema inference doesn't see it — + * surface it in the test form so a partitioned script can be run manually. + */ +export function injectPartitionArg( + s: any, + a: Record | undefined, + l: string | undefined, + c: string +) { + try { + if (l !== 'duckdb' || !s?.properties) return + const part = parsePipelineAnnotations(c).partition + if (!part) return + // Date-based partition kinds render a date / datetime picker; a dynamic + // key is a free-form string. + const format = + part.kind === 'hourly' + ? 'date-time' + : part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly' + ? 'date' + : undefined + if (!s.properties['partition']) { + s.properties['partition'] = { + type: 'string', + ...(format ? { format } : {}), + // ISO output so partition keys sort lexicographically (the date + // picker defaults to dd-MM-yyyy otherwise). + ...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}), + description: + part.kind === 'dynamic' + ? 'Partition key value to materialize.' + : `Partition (${part.kind}) to materialize.` + } + if (Array.isArray(s.order) && !s.order.includes('partition')) { + s.order = ['partition', ...s.order] + } + } + // Pre-fill the *test* arg with the current slice for date kinds — a + // convenience default, kept on the args (not baked into the schema, + // where it would persist to the deployed script and go stale). + if (format && a && (a['partition'] == null || a['partition'] === '')) { + const now = new Date() + a['partition'] = + format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16) + } + } catch (e) {} +} + +/** + * The schema the script editor holds right after mounting on `content`, without + * any user input: `ScriptEditor.inferSchema` re-infers the main signature into + * the stored schema and injects the duckdb partition arg. A schema stored by + * another producer (a CLI push, an older parser) can lack keys the current + * parser emits, so the editor rewrites it on mount; a draft baseline taken from + * the raw stored schema would then never compare equal to the untouched editor. + * Returns a copy; a parse failure leaves it as far as inference got, which is + * also what the editor keeps. + */ +export async function schemaAsEditorMounts( + language: SupportedLanguage | undefined, + content: string, + schema: unknown, + kind: Script['kind'] | undefined +): Promise { + const copy: Schema = schema ? structuredClone(schema as Schema) : emptySchema() + const infer = async () => { + await inferArgs(language, content, copy, kind === 'preprocessor' ? 'preprocessor' : undefined) + injectPartitionArg(copy, undefined, language, content) + } + try { + await infer() + } catch { + // The editors retry a failed mount inference once (transient wasm init); + // the baseline has to land where that retry leaves the editor. + if (!content || !language) return copy + try { + await infer() + } catch {} + } + return copy +} diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 76ecfa59d8..b028e9360d 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -1370,6 +1370,11 @@ test_behavior: build vars: {} threads: 4 full_refresh: false +# Resolve a ref() this run does not build through the state the last successful +# run of this environment published, instead of through the schema it writes +# into. The default for the run form's toggle: the run that publishes the state +# and the run that defers to it are two invocations of this one script. +defer: false # Rebuild the nodes a failed build left failed or skipped, in this same job, # before reporting failure. dbt confines a failure to its own subtree, so a # transient warehouse error costs those nodes rather than the whole project. @@ -1381,6 +1386,12 @@ full_refresh: false # resolved to that Windmill variable, so secrets stay out of this file. # env: # DBT_PASSWORD: $var:u/user/my_warehouse_password +# Real column schemas — every column typed and in the order the model produces +# it — from the engine's static analysis, which also records column-level +# lineage for a later view. Opt-in because it runs a separate dbt compile under +# --static-analysis strict, which rejects SQL the default accepts; a project it +# cannot analyze keeps the graph it has. Needs an engine that computes it. +# column_lineage: true ` // for related places search: ADD_NEW_LANG export const INITIAL_CODE = { diff --git a/frontend/src/lib/userDraft.svelte.ts b/frontend/src/lib/userDraft.svelte.ts index 24572ee1ab..57e911dd3c 100644 --- a/frontend/src/lib/userDraft.svelte.ts +++ b/frontend/src/lib/userDraft.svelte.ts @@ -219,7 +219,10 @@ const DRAFT_COMPARE_IGNORED_FIELDS = [ 'version_id', 'parent_version', 'is_draft', - 'assets' + 'assets', + // Fixed at creation and absent from the resource editor's draft shape, so + // it only ever shows up on the deployed side of a comparison. + 'resource_type' ] as const /** @@ -349,6 +352,32 @@ export const UserDraft = { void UserDraftDbSyncer.save({ workspace: ws, itemKind, path, value: null }) }, + /** + * Drop this key's in-memory cell and cached write WITHOUT touching the + * server. For callers that have already deleted the row by another route + * (a deploy) and only need the local mirror to stop answering `get`/`has` + * with a value that is gone. + * + * MUST be used instead of `remove` there. `remove` POSTs its own + * `value: null`, and that POST is debounced and carries whatever + * `last_sync` is left — which a preceding successful delete has already + * cleared. The backend treats a delete with no `last_sync` as + * unconditional, so the second POST lands ~1.5s later with nothing to + * compare against and removes a draft saved in the meantime. + */ + forgetLocal(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { + const ws = resolveWorkspace(opts) + const mk = mapKey(ws, itemKind, path) + const entry = entries.get(mk) + if (entry) { + // Same as `remove`: clear the cell so live observers see the delete, and arm + // `skipNextSync` so the mirror does not turn that write into a POST of its own. + entry.skipNextSync = true + entry.state.val = undefined + } + writtenCache.delete(mk) + }, + clear(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { UserDraft.discard(itemKind, path, undefined, opts) }, diff --git a/frontend/src/lib/userDraftEditGate.ts b/frontend/src/lib/userDraftEditGate.ts new file mode 100644 index 0000000000..e3057a73ee --- /dev/null +++ b/frontend/src/lib/userDraftEditGate.ts @@ -0,0 +1,66 @@ +import { onDestroy } from 'svelte' + +/** + * What kind of event opened the gate. + * + * - `value`: the user changed something — the event *is* the edit. + * - `precursor`: a gesture that usually precedes an edit. Needed because a + * custom component (a picker, a toggle built out of divs) writes its value + * through Svelte state and fires no native value event at all, so waiting for + * one would drop those edits. The cost is that a bare click counts too. + */ +export type UserInputKind = 'value' | 'precursor' + +/** Events that ARE an edit. `drop` and `paste` matter on their own: text + * dragged in from another application produces no pointer or key event in this + * document at all. */ +const VALUE_EVENTS = ['input', 'change', 'drop', 'paste'] as const +/** `click` is here for the controls that mutate state from a click handler and + * fire no native value event — ArgInput's "Add item", say. A mouse always sends + * `pointerdown` first, but an assistive technology can activate one with a + * trusted `click` alone, and that is a real edit with nothing else to catch it. */ +const PRECURSOR_EVENTS = ['pointerdown', 'keydown', 'click'] as const + +/** + * A draft is supposed to record what the USER changed, but an editor built + * from a schema writes into the value on its own: the form materializes a + * property the stored item never carried (an empty string, `false`, the first + * option of a required enum, a schema `default`) and deletes one a `showExpr` + * hides. So merely opening an item whose schema has moved on makes it diverge + * from the deployed value with nobody having touched it — a draft nobody asked + * for, cluttering the workspace. + * + * An editor guards against that by gating its draft on this: nothing the form + * settles on counts until the user has actually put something in. Callers + * decide what a gate covers (the resource editor keys it by workspace, since + * switching workspaces re-renders the form against a fresh value) and what + * gating means for them — suspending the autosave, absorbing the settled value + * into the deployed baseline, or both. + * + * Capture phase puts this ahead of the handler that writes the value, so a gate + * opened here is already open by the time the edit lands. Listening on the + * document rather than the editor's own subtree is deliberate: pickers and + * modals render in portals outside it, and missing a real edit would silently + * drop the user's work, while opening the gate too eagerly only costs the + * phantom draft that existed before. + * + * Registers for the lifetime of the calling component — call it during init. + */ +export function onUserInput(handle: (kind: UserInputKind) => void): void { + if (typeof document === 'undefined') return + const listeners: Array<[string, (e: Event) => void]> = [] + const register = (type: string, kind: UserInputKind) => { + const onEvent = (e: Event) => { + // A programmatic `dispatchEvent` is untrusted, which is what keeps the + // form's own settling from opening the gate it is gated by. + if (e.isTrusted) handle(kind) + } + document.addEventListener(type, onEvent, true) + listeners.push([type, onEvent]) + } + for (const type of VALUE_EVENTS) register(type, 'value') + for (const type of PRECURSOR_EVENTS) register(type, 'precursor') + onDestroy(() => { + for (const [type, onEvent] of listeners) document.removeEventListener(type, onEvent, true) + }) +} diff --git a/frontend/src/lib/userDraftPrune.test.ts b/frontend/src/lib/userDraftPrune.test.ts new file mode 100644 index 0000000000..0276598d45 --- /dev/null +++ b/frontend/src/lib/userDraftPrune.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' + +// The sweep DELETES drafts, so the guard that matters is which rows it picks. +// Stub the two collaborators it decides from — the draft listing and the +// per-kind diff — and assert on what it discards. +const listDrafts = vi.fn() +const getDraftDiffValues = vi.fn() +const updateDraft = vi.fn(async () => ({ status: 'saved', current_timestamp: 'x' })) + +vi.mock('./gen', () => ({ + DraftService: { + listDrafts: (...a: unknown[]) => listDrafts(...(a as [])), + updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) + } +})) +// Only `getDraftDiffValues` is stubbed; `canDiffDraftKind` is the real one, so +// the kind filter is pinned against the actual overlay table. +vi.mock('./utils_draft_deploy', async (orig) => ({ + ...(await orig>()), + getDraftDiffValues: (...a: unknown[]) => getDraftDiffValues(...(a as [])) +})) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) +vi.mock('./workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) +const sendUserToast = vi.fn() +vi.mock('./toast', () => ({ sendUserToast: (...a: unknown[]) => sendUserToast(...(a as [])) })) + +// The sweep reads exactly one thing from the syncer — whether this tab is +// mid-write on the key — and writes nothing back to it. +let syncState = 'none' +vi.mock('./userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { + getState: () => ({ + get state() { + return syncState + } + }) + } +})) + +let liveDraft = false +vi.mock('./userDraft.svelte', async (orig) => ({ + ...(await orig>()), + UserDraft: { has: () => liveDraft } +})) + +import { pruneMeaninglessDrafts } from './userDraftPrune' + +const row = (over: Record = {}) => ({ + kind: 'resource', + path: 'u/me/r', + draft_only: false, + legacy_draft: false, + mine: true, + can_write: true, + created_at: '2026-01-01T00:00:00Z', + ...over +}) +const diff = (over: Record = {}) => ({ + deployed: { value: { host: 'h' } }, + draft: { value: { host: 'h' } }, + hasDraft: true, + noDeployed: false, + ...over +}) +const discardedPaths = () => updateDraft.mock.calls.map((c: any[]) => c[0].path as string) + +beforeEach(() => { + localStorage.clear() + vi.clearAllMocks() + updateDraft.mockResolvedValue({ status: 'saved', current_timestamp: 'x' }) + syncState = 'none' + liveDraft = false +}) + +describe('pruneMeaninglessDrafts', () => { + it('discards a draft whose diff against the deployed value is empty', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('keeps a draft that carries a real change', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff({ draft: { value: { host: 'other' } } })) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches a draft-only item — the draft is the whole item', async () => { + listDrafts.mockResolvedValue([row({ draft_only: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches another user’s row, or one it cannot write', async () => { + listDrafts.mockResolvedValue([row({ mine: false }), row({ path: 'u/me/b', can_write: false })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves a draft alone when its diff cannot be fetched, and retries it later', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('boom')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // A row that could not be judged is not a row that carries changes, so + // the pass must stay open rather than strand it. + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('conditions the delete on the timestamp it judged, so a row that moved is spared', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockResolvedValue({ status: 'conflict', current_timestamp: 'newer' }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledWith({ + workspace: 'main', + kind: 'resource', + path: 'u/me/r', + requestBody: { value: null, last_sync: '2026-01-01T00:00:00Z', force: false } + }) + // Refused, so nothing is reported as cleared — and the sweep leaves no + // state behind for the editor's own autosave to trip over. + expect(sendUserToast).not.toHaveBeenCalled() + }) + + it('does not keep retrying a row the server will never judge', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue({ status: 404 }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // Sealed: a 4xx is final, unlike the transient case above. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('gives up after a bounded number of unresolved passes', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('network')) + for (let i = 0; i < 3; i++) await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + // Sealed on the third: an unresolvable row cannot re-list forever. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + }) + + it('skips a kind no diff can be computed for, and still seals', async () => { + listDrafts.mockResolvedValue([row({ kind: 'trigger_webhook', path: 'u/me/hook' })]) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(getDraftDiffValues).not.toHaveBeenCalled() + // Unjudgeable is permanent, not transient: leaving the pass open would + // re-run the sweep on every page load forever. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft this tab is editing', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft with a write queued or in flight', async () => { + syncState = 'pending' + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('does not count, or seal the pass on, a delete that failed to send', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockRejectedValueOnce(new Error('network')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(sendUserToast).not.toHaveBeenCalled() + // The pass stayed open, so the draft left behind is retried. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r', 'u/me/r']) + }) + + it('never touches a legacy workspace-level row', async () => { + listDrafts.mockResolvedValue([row({ legacy_draft: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('only sweeps the kinds whose editors are gated', async () => { + listDrafts.mockResolvedValue([ + row({ kind: 'script', path: 'u/me/s' }), + row({ kind: 'flow', path: 'u/me/f' }), + row({ kind: 'app', path: 'u/me/a' }), + row({ kind: 'trigger_schedule', path: 'u/me/sched' }), + row() + ]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths().sort()).toEqual(['u/me/r', 'u/me/sched']) + // The expensive payload fetches are never made for the ungated kinds. + expect(getDraftDiffValues).toHaveBeenCalledTimes(2) + }) + + it('leaves the pass open when a row was skipped as busy', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + liveDraft = false + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('runs once per workspace and user', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(1) + await pruneMeaninglessDrafts('other', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(2) + }) + + it('retries next mount when the listing failed', async () => { + listDrafts.mockRejectedValueOnce(new Error('offline')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) +}) diff --git a/frontend/src/lib/userDraftPrune.ts b/frontend/src/lib/userDraftPrune.ts new file mode 100644 index 0000000000..465048d09d --- /dev/null +++ b/frontend/src/lib/userDraftPrune.ts @@ -0,0 +1,230 @@ +/** + * One-off sweep that drops drafts carrying no changes. + * + * `onUserInput` stops new ones from being written; the ones already stored need + * this pass to clear. Runs once per (workspace, user) per browser, after the + * localStorage→DB migration so anything it just uploaded is swept too. + * + * Scoped to the kinds whose editors that gate covers. A script, flow or app + * draft only ever came from an explicit edit, so there is no phantom to clear + * there — and `getDraftDiffValues` would fetch each one's full deployed payload + * at login to prove it. + * + * A draft is dropped only when the diff the user would be shown is empty: both + * sides come from `getDraftDiffValues`, the same canonicalization the diff + * drawer renders, compared with the same `draftValuesEqual` the editors use. + * Anything that can't be established is left alone — a `draft_only` item (no + * deployed counterpart, so discarding would destroy the item itself), a kind + * with no diff support, a failed fetch, and the legacy workspace-level rows, + * which belong to nobody and are admin-gated to migrate. + * + * Deleting is the dangerous half, and the equality behind it is always stale: + * it was read one round trip ago, and every candidate is read before any is + * deleted. So the delete is a compare-and-delete — `last_sync` is the + * timestamp the row was judged on, and the backend drops it only if nothing has + * written it since, whoever wrote it. + * + * It is sent straight to `DraftService`, NOT through `UserDraftDbSyncer`. That + * syncer exists to autosave an editor's own live value, and everything it does + * for that — parking the payload for the `pagehide` flush, debouncing, holding + * a per-tab `last_sync` baseline and conflict state — is a way for a one-shot + * delete to reach back into whatever the user is doing in the same tab. This + * sweep wants exactly one conditional request and no state afterwards. + */ + +import { DraftService } from './gen' +import type { UserDraftItemKind } from './gen' +import { sendUserToast } from './toast' +import { setLocalDraftHint } from './localDraftHints.svelte' +import { UserDraft, draftValuesEqual } from './userDraft.svelte' +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' +import { canDiffDraftKind, getDraftDiffValues } from './utils_draft_deploy' +import { invalidateWorkspaceDrafts } from './workspaceDrafts.svelte' + +const SENTINEL_PREFIX = 'userdraft/pruned/v1/' + +/** A pass that leaves anything unresolved runs again next mount, which costs a + * listing plus an overlay GET per row. Bounded so no permanently-unresolvable + * row can make that repeat forever — whatever the reason it can't be judged. */ +const MAX_PASSES = 3 + +/** The editors whose forms are built from a schema, and so the only kinds that + * could have banked a draft nobody wrote — minus the ones no diff can be + * computed for, which would throw and keep the pass unsealed forever. */ +function isSweepableKind(kind: UserDraftItemKind): boolean { + return (kind === 'resource' || kind.startsWith('trigger_')) && canDiffDraftKind(kind) +} + +/** Overlay GETs are one round trip each and a cluttered workspace has dozens; + * a small window keeps the sweep off the critical path of a fresh login. */ +const CONCURRENCY = 4 + +/** Guards against the layout effect firing again before the sentinel lands. */ +const inFlight = new Set() + +type Candidate = { + kind: UserDraftItemKind + path: string + /** The row's `created_at` as listed — the baseline the delete is conditioned on. */ + createdAt: string +} + +const attemptsKey = (sentinel: string) => `${sentinel}:attempts` + +function readAttempts(sentinel: string): number { + try { + const n = Number(localStorage.getItem(attemptsKey(sentinel))) + return Number.isFinite(n) && n > 0 ? n : 0 + } catch { + return 0 + } +} + +/** Is this tab holding or writing this draft right now? */ +function busyLocally(workspace: string, kind: UserDraftItemKind, path: string): boolean { + if (UserDraft.has(kind, path, { workspace })) return true + return UserDraftDbSyncer.getState({ workspace, itemKind: kind, path }).state !== 'none' +} + +/** A 4xx is the server's final answer for this row — the item is gone, or the + * kind's overlay endpoint isn't served by this build (a feature-gated trigger + * on CE). Retrying it on every page load would never succeed. Anything else + * (network, 5xx) is worth another pass. 429 asks for exactly that. */ +function isPermanentlyUnjudgeable(e: unknown): boolean { + const status = (e as { status?: unknown })?.status + return typeof status === 'number' && status >= 400 && status < 500 && status !== 429 +} + +/** `undefined` when the diff could not be fetched and might be next time — + * distinct from `false`, so the caller can leave the pass open rather than + * strand a row it never judged. */ +async function carriesNoChanges( + workspace: string, + { kind, path }: Candidate +): Promise { + try { + const { deployed, draft, hasDraft, noDeployed } = await getDraftDiffValues( + kind, + path, + workspace + ) + // `hasDraft` false means the overlay had no draft row and the item's own + // value stood in for the draft side — there is nothing to discard, and the + // two sides would compare equal by construction. + if (!hasDraft || noDeployed) return false + return draftValuesEqual(draft, deployed) + } catch (e) { + return isPermanentlyUnjudgeable(e) ? false : undefined + } +} + +async function mapWithLimit( + items: T[], + limit: number, + fn: (item: T) => Promise +): Promise { + const out = new Array(items.length) + let next = 0 + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const i = next++ + out[i] = await fn(items[i]) + } + }) + ) + return out +} + +export async function pruneMeaninglessDrafts(workspace: string, userKey: string): Promise { + if (typeof localStorage === 'undefined') return + const sentinel = `${SENTINEL_PREFIX}${workspace}/${userKey}` + if (inFlight.has(sentinel)) return + try { + if (localStorage.getItem(sentinel)) return + } catch { + // Storage unavailable (private mode): the sweep can't record that it ran, + // and re-running it on every mount would cost an overlay GET per draft. + return + } + inFlight.add(sentinel) + try { + const rows = await DraftService.listDrafts({ workspace }) + // Anything left unresolved keeps the pass open: a row skipped as busy was + // never judged, and one whose delete failed is still there. Sealing on + // either would strand it. + let unresolved = 0 + const candidates: Candidate[] = rows + // `draft_only` rows ARE the item; `mine` / `can_write` are the same + // gate the discard endpoint enforces, so anything else would 403. + .filter((r) => !r.draft_only && r.mine && r.can_write) + .filter((r) => isSweepableKind(r.kind) && !r.legacy_draft) + .filter((r) => { + if (!busyLocally(workspace, r.kind, r.path)) return true + unresolved++ + return false + }) + .map((r) => ({ + kind: r.kind, + path: r.path, + createdAt: r.created_at + })) + + const empty: Candidate[] = [] + await mapWithLimit(candidates, CONCURRENCY, async (c) => { + const verdict = await carriesNoChanges(workspace, c) + if (verdict === undefined) unresolved++ + else if (verdict) empty.push(c) + }) + + let discarded = 0 + for (const c of empty) { + // Re-check: the reads above took a while, and the user may have opened + // this item in the meantime. + if (busyLocally(workspace, c.kind, c.path)) { + unresolved++ + continue + } + try { + const resp = await DraftService.updateDraft({ + workspace, + kind: c.kind, + path: c.path, + requestBody: { value: null, last_sync: c.createdAt, force: false } + }) + // `conflict` means the row moved past the timestamp we judged it on, + // so it is no longer the empty draft we decided to drop. + if (resp.status === 'saved') { + setLocalDraftHint(workspace, c.kind, c.path, false) + discarded++ + } + } catch { + unresolved++ + } + } + if (discarded > 0) { + invalidateWorkspaceDrafts(workspace) + sendUserToast(`Cleared ${discarded} draft${discarded > 1 ? 's' : ''} that carried no changes`) + } + // Seal once nothing is left hanging, or once we have tried enough times + // that whatever is hanging is not going to resolve. + const attempts = readAttempts(sentinel) + 1 + if (unresolved === 0 || attempts >= MAX_PASSES) { + try { + localStorage.setItem(sentinel, new Date().toISOString()) + localStorage.removeItem(attemptsKey(sentinel)) + } catch { + // Nothing to do — the pass is idempotent, it just runs again. + } + } else { + try { + localStorage.setItem(attemptsKey(sentinel), String(attempts)) + } catch {} + } + } catch { + // Fire-and-forget from the layout: a workspace whose draft list can't be + // read is left exactly as it was. + } finally { + inFlight.delete(sentinel) + } +} diff --git a/frontend/src/lib/utils_draft_deploy.test.ts b/frontend/src/lib/utils_draft_deploy.test.ts index 9bed95917b..4495155f24 100644 --- a/frontend/src/lib/utils_draft_deploy.test.ts +++ b/frontend/src/lib/utils_draft_deploy.test.ts @@ -7,7 +7,7 @@ vi.mock('$lib/gen', () => ({ DraftService: { deleteDraft: vi.fn() }, AppService: {}, VariableService: {}, - ResourceService: {}, + ResourceService: { getResource: vi.fn(), updateResource: vi.fn(), createResource: vi.fn() }, ScheduleService: {}, HttpTriggerService: {}, WebsocketTriggerService: {}, @@ -21,7 +21,9 @@ vi.mock('$lib/gen', () => ({ AzureTriggerService: {}, EmailTriggerService: {} })) -vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ UserDraftDbSyncer: { save: vi.fn() } })) +vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { save: vi.fn(), recordRemoteSync: vi.fn() } +})) vi.mock('$lib/workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) vi.mock('$lib/workspaceComparison', () => ({ invalidateWorkspaceComparison: vi.fn() })) vi.mock('$lib/localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) @@ -30,7 +32,8 @@ vi.mock('$lib/components/raw_apps/utils', () => ({ canonicalRawAppDiffValue: vi. vi.mock('$lib/appDiffSides', () => ({ classicAppDraftParts: vi.fn() })) vi.mock('$lib/utils_deployable', () => ({ TRIGGER_RUNTIME_IGNORE: [] })) -import { ScriptService, FlowService } from '$lib/gen' +import { ScriptService, FlowService, ResourceService } from '$lib/gen' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' // draftBaseIsStale compares a draft's base pointer against the deployed head // of the item it was fetched with (`get_draft=true`). Shared by CompareDrafts @@ -124,3 +127,65 @@ describe('deployDraft preserves on_behalf_of', () => { ) }) }) + +// The resource branch reads the item again when the deploy lands, and falls back to the deployed +// row when the draft has gone. That row keeps its value under `value` and carries no `args` at +// all, so reading it as a draft (`value: d.args ?? {}`) would replace a live resource with `{}`. +describe('deployDraft: resource with no draft', () => { + beforeEach(() => vi.clearAllMocks()) + + it('writes nothing rather than `{}` over the deployed value', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' } + } as any) + + // `noop` is what lets a caller deploying one specific draft tell "nothing to promote" apart + // from "deployed", instead of reporting an agent as deployed that was never written. + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ + success: true, + noop: true + }) + expect(ResourceService.updateResource).not.toHaveBeenCalled() + expect(ResourceService.createResource).not.toHaveBeenCalled() + // Nor does it touch the draft row. There was none of this user's to delete, so the only row + // the cleanup could reach is one written after the read: an edit destroyed without ever + // having been deployed. Clearing the baseline would be the same bug by another route, since + // a delete with no baseline is the unconditional one. + expect(UserDraftDbSyncer.save).not.toHaveBeenCalled() + expect(UserDraftDbSyncer.recordRemoteSync).not.toHaveBeenCalled() + }) + + it('still deploys normally when the draft is there, and keys the cleanup to the row it read', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' }, + draft_saved_at: '2026-01-01T00:00:00Z', + draft: { path: 'f/support/triage_agent', args: { system_prompt: 'drafted' } } + } as any) + + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ success: true }) + expect(ResourceService.updateResource).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ value: { system_prompt: 'drafted' } }) + }) + ) + // The draft delete that follows is conditional on this baseline. With no baseline the backend + // deletes unconditionally, destroying a draft saved between the read and the delete without + // ever having deployed it, so the timestamp has to be the one from the row just promoted. + expect(UserDraftDbSyncer.recordRemoteSync).toHaveBeenCalledWith( + { workspace: 'ws', itemKind: 'resource', path: 'f/support/triage_agent' }, + '2026-01-01T00:00:00Z' + ) + expect(UserDraftDbSyncer.save).toHaveBeenCalledWith( + expect.objectContaining({ path: 'f/support/triage_agent', value: null, immediate: true }) + ) + // Order is the whole point: a delete issued before the seed carries whatever baseline the tab + // happened to hold, which for a caller that only read through a listing is none at all. + expect(vi.mocked(UserDraftDbSyncer.recordRemoteSync).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(UserDraftDbSyncer.save).mock.invocationCallOrder[0] + ) + }) +}) diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index b322d6adc4..0d121d0627 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -95,6 +95,19 @@ const OVERLAY_GETTERS: Partial< EmailTriggerService.getEmailTrigger({ workspace, path, getDraft: true }) } +/** Whether `getDraftDiffValues` can produce a diff for this kind at all. The + * script/flow/app family is handled inline; every other kind needs an overlay + * getter and throws without one — several trigger kinds have none. */ +export function canDiffDraftKind(kind: DraftKind): boolean { + return ( + kind === 'script' || + kind === 'flow' || + kind === 'app' || + kind === 'raw_app' || + OVERLAY_GETTERS[kind] !== undefined + ) +} + /** Strip the per-user draft-overlay metadata, returning `{deployed, draft}`. */ function splitOverlay(r: any): { deployed: any @@ -435,8 +448,13 @@ export async function deployDraft( path: string, workspace: string, opts: { draftOnly?: boolean; rawApp?: boolean; deploymentMessage?: string } = {} -): Promise { +): Promise { const { draftOnly = false, rawApp = false, deploymentMessage } = opts + // Set when the branch found nothing to promote and wrote nothing. Success, because the item is + // already at the value a deploy would have left it at and its stale draft state still wants + // clearing — but a caller deploying one specific draft it showed the user has to be able to tell + // that apart from having deployed it. + let noop = false try { if (kind === 'raw_app' || (kind === 'app' && rawApp)) { // Raw apps bundle their source files and deploy via the raw-app @@ -579,10 +597,34 @@ export async function deployDraft( } void deployed } else if (kind === 'resource') { - const { deployed, draft: d } = splitOverlay(await OVERLAY_GETTERS.resource!(workspace, path)) + const overlay = await OVERLAY_GETTERS.resource!(workspace, path) + // Adopt the row this promote is based on as the baseline for the delete below. Without one + // the backend deletes unconditionally, so a draft saved between that read and the delete is + // destroyed having never been deployed — a caller that only ever read through a listing has + // no baseline of its own to supply. With it the delete is refused instead and the newer + // draft survives, which is the recoverable outcome of the two. Only ever seeded, never + // cleared: passing no timestamp drops whatever baseline the tab already held, which would + // turn that same delete back into an unconditional one. + if (overlay?.draft_saved_at) { + UserDraftDbSyncer.recordRemoteSync( + { workspace, itemKind: kind, path }, + overlay.draft_saved_at + ) + } + const { deployed, draft: d, hasDraft } = splitOverlay(overlay) // ResourceEditor's `ResourceState` draft shape: // { path, description, args, resource_type?, labels?, wsSpecific } - if (draftOnly) { + // The deployed row is a different shape (`value`, `ws_specific`, no `args` at all), and + // `splitOverlay` hands it back as the draft side when the draft row has gone — deployed or + // discarded from another tab between the listing and this click. Reading it as a draft is + // what made `value: d.args ?? {}` replace a live resource with `{}`. Nothing to promote + // then, so write nothing and fall through to the cleanup below, which clears the stale + // local draft hint and the drafts listing. The item is already at the value a successful + // deploy would have left it at, so this reports success rather than an error, matching + // what the other kinds end up doing when their own draft is gone. + if (!hasDraft) { + noop = true + } else if (draftOnly) { await ResourceService.createResource({ workspace, requestBody: { @@ -633,8 +675,8 @@ export async function deployDraft( return { success: false, error: `Deploy not supported for draft kind ${kind}` } } // Delete the draft at its STORAGE path (the row key, = the `path` arg). - // Two reasons it must happen here for every kind, mirroring the editors' - // post-deploy `discardDraftAfterDeploy(draftPath)`: + // Two reasons it must happen here for every kind that promoted something, + // mirroring the editors' post-deploy `discardDraftAfterDeploy(draftPath)`: // - Drawer kinds (variable / resource / triggers) aren't deleted by // their create/update endpoints at all. // - script/flow/app/raw_app DO delete server-side, but only the draft at @@ -642,13 +684,18 @@ export async function deployDraft( // synthetic `u/{user}/draft_{uuid}` storage path ≠ `d.path`, so its // draft row survives the deploy and keeps listing. Deleting the // storage-path draft removes it (a no-op when the server already did). - await UserDraftDbSyncer.save({ - workspace, - itemKind: kind, - path, - value: null, - immediate: true - }) + // Skipped when nothing was promoted: the read that set `noop` found no draft of this user's to + // delete, so the only row this could reach is one written after it — destroying an edit that + // was never deployed, and never even listed. + if (!noop) { + await UserDraftDbSyncer.save({ + workspace, + itemKind: kind, + path, + value: null, + immediate: true + }) + } // Mutated the workspace's Server Drafts — refresh every mounted reader. invalidateWorkspaceDrafts(workspace) // The DEPLOYED state moved: cached fork comparisons involving this @@ -659,7 +706,7 @@ export async function deployDraft( // so the syncer-owned hint won't auto-clear — clear it explicitly. // (Idempotent: the drawer-kind delete above already cleared it.) setLocalDraftHint(workspace, kind, path, false) - return { success: true } + return noop ? { success: true, noop: true } : { success: true } } catch (e: any) { return { success: false, error: e?.body ?? e?.message ?? String(e) } } diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 69b6217a73..d5ca0d5167 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -73,6 +73,7 @@ import { createUsageResources, registerUsageResources } from '$lib/usage.svelte' import { purgeLegacyUserDrafts } from '$lib/userDraftLegacyMigration' import { migrateUserDraftsToDb } from '$lib/userDraftDbMigration' + import { pruneMeaninglessDrafts } from '$lib/userDraftPrune' import DraftMigrationErrorModal from '$lib/components/DraftMigrationErrorModal.svelte' import { onDestroy, setContext, untrack } from 'svelte' import { base } from '$app/paths' @@ -790,12 +791,16 @@ // drafts). `migrateUserDraftsToDb` then pushes the workspace-scoped // `userdraft/w/{ws}/{kind}/{path}` keys — written by the editor with the // correct workspace — onto the server-side draft table, clearing LS on - // success. + // success. `pruneMeaninglessDrafts` then clears the drafts an older, stricter + // comparison saved for changes nobody made; it runs after the upload so the + // entries that just landed are swept in the same pass. $effect(() => { - if ($workspaceStore && $userStore) { + const ws = $workspaceStore + const email = $userStore?.email + if (ws && email) { untrack(() => { purgeLegacyUserDrafts() - void migrateUserDraftsToDb() + void migrateUserDraftsToDb().then(() => pruneMeaninglessDrafts(ws, email)) }) } }) diff --git a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte index 0af3cdeded..4ff8670cda 100644 --- a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte @@ -1,11 +1,11 @@