diff --git a/.agents/skills/update-sqlx/SKILL.md b/.agents/skills/update-sqlx/SKILL.md index ce4eaa5ff7..ccbd7baf3d 100644 --- a/.agents/skills/update-sqlx/SKILL.md +++ b/.agents/skills/update-sqlx/SKILL.md @@ -9,11 +9,75 @@ Windmill uses `SQLX_OFFLINE=true` in CI, which requires all `sqlx::query!` / `sq ## When to Run -Run after any change to SQL queries in Rust source files. Without it, CI will fail with: +Run after **adding or editing** a SQL query in Rust source. Without it, CI fails with: ``` error: `SQLX_OFFLINE=true` but there is no cached data for this query ``` +**Do NOT run it when a change only *removes* queries.** The cache is already complete for +CI; all that is left are orphaned entries, which are cosmetic and never break a build. +Running `prepare` to tidy them risks destroying the cache for no gain. Delete them +offline instead: for each `.sqlx/query-*.json`, normalize its `query` field (strip `\` +line-continuations, collapse whitespace) and check whether it still appears in any `.rs` +file. That detector reports ~48 false positives in a CE checkout — EE queries live in +`*_ee.rs` symlinks it cannot read — so **filter to the tables your change touched** and +delete only those. + +## Before You Run Anything + +1. **Back the cache up.** `prepare` deletes `.sqlx/` *before* regenerating, so any compile + failure leaves it gutted (observed: 2350 → 142 entries). + ```bash + cp -r backend/.sqlx /tmp/sqlx_backup # restore with: rm -rf backend/.sqlx && cp -r /tmp/sqlx_backup backend/.sqlx + ``` +2. **Point `DATABASE_URL` at THIS worktree's database.** `prepare` compiles every + `sqlx::query!` against the **live** database. Another worktree's DB lacks your + migrations, so every new-table query fails and takes the cache down with it. The + symptom is `relation "" does not exist` — that is a wrong + `DATABASE_URL`, not a broken query. See AGENTS.md → "Per-worktree ports and database". + +## Queries Inside Tests Need `--all-targets`, Which Fails In A CE Checkout + +`prepare` only caches queries in code it compiles, and `--workspace` alone does **not** +compile test targets. A `sqlx::query!` inside `tests/*.rs` therefore gets no entry, and CI +fails on the test target with the usual "no cached data" error even though the lib built +clean. `SQLX_OFFLINE=true cargo check --workspace --all-targets` is what reproduces it. + +Adding `--all-targets` caches them — and, in a CE checkout, **aborts partway through**: +`backend/tests/otel.rs` imports `windmill_common::otel_ee`, which exists only behind the +`private` feature, so the compile dies after `prepare` has already emptied `.sqlx/`. +Observed: 2435 → 4 entries, `error: cargo check failed with status: exit status: 101`. + +Do not fight it — the abort is a pre-existing EE gap, not something your change caused. +Take the entries you need and put the backup back: + +```bash +cd backend +cp -r .sqlx /tmp/sqlx_backup +ls /tmp/sqlx_backup | sort > /tmp/before.txt + +DATABASE_URL= \ + cargo sqlx prepare --workspace -- --workspace --features all_sqlx_features --all-targets +# expected to fail; it still wrote the entries it got to before dying + +ls .sqlx | sort > /tmp/after.txt +mkdir -p /tmp/newq +comm -13 /tmp/before.txt /tmp/after.txt | while read f; do cp ".sqlx/$f" /tmp/newq/; done + +rm -rf .sqlx && cp -r /tmp/sqlx_backup .sqlx && cp /tmp/newq/*.json .sqlx/ +``` + +**Read every file in `/tmp/newq` before copying it in** — print each one's `query` field and +confirm it is one of yours. The set is small (one per new test query), and anything else in +there means the run got further than you think. + +Then verify both targets, since the lib passing says nothing about the tests: + +```bash +SQLX_OFFLINE=true cargo check --workspace --features all_sqlx_features # lib +SQLX_OFFLINE=true cargo check -p --all-targets # tests +``` + ## The Problem `cargo sqlx prepare --workspace` **deletes all existing cache files** and regenerates only the ones found in the current compilation. If you don't compile with every feature flag (especially `private` for EE files), you will **silently delete EE query caches**, breaking CI for enterprise tests. @@ -68,7 +132,16 @@ But if it fails with EE compilation errors, use the safe procedure above. - **Never** run `cargo sqlx prepare --workspace` with only OSS features and commit the result — it will delete EE caches. - **Never** set `SQLX_OFFLINE=true` for local `cargo sqlx prepare` — use a live database per CLAUDE.md. (CI runs with `SQLX_OFFLINE=true`, which is why the cache must be complete.) +- **Never** run `prepare` without a `.sqlx` backup, or against a `DATABASE_URL` you have not confirmed belongs to this worktree. +- **Never** run `prepare` at all for a removal-only change. - **Never** skip the verification step (step 4 above). +- **Never** leave a `--all-targets` run's output in place after it aborts — it is a + near-empty cache. Restore the backup and graft on only the entries you verified. + +Step 4 compares against `origin/main` because step 1 restored from it, so the two agree. +If you did **not** run step 1 — auditing a branch's cache on its own, say — compare +against `git merge-base HEAD origin/main` instead: `origin/main` advances, so its newer +entries would read as losses on your branch. ## Verification diff --git a/AGENTS.md b/AGENTS.md index 90c0060ae2..31c8a93de3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,9 @@ Open-source platform for internal tools, workflows, API integrations, background - **Validation**: `docs/validation.md` — what checks to run based on what you changed - **Unreleased SDK changes**: `docs/wac-sdk-e2e.md` — exercising a client change on a real worker +- **Agent workers**: `docs/agent-worker-e2e.md` — building and running one locally. An agent + reaches the DB only through the API, so `Connection::Http` paths are never taken by a plain + `cargo run`; a normal build cannot start one at all. - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. @@ -24,6 +27,11 @@ Open-source platform for internal tools, workflows, API integrations, background ## Dev Environment +> **In a git worktree, the ports and database below are NOT the ones to use.** Each +> worktree gets its own backend port, frontend port and Postgres database, so the +> defaults in this section apply only to a plain single checkout. **Discover the real +> values before running anything** — see "Per-worktree ports and database" below. + - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) - **DuckDB local jobs**: before running DuckDB scripts locally, build the FFI shared library with `cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. Re-run it after clean builds or when `backend/target/debug/libwindmill_duckdb_ffi_internal.*` is missing. The bundled DuckDB compile (~2min) is cached in a per-user dir shared across worktrees, so a fresh worktree reuses it and the build is near-instant. - **Data pipelines (DuckLake) from source**: a plain `cargo run` (even `--features quickjs`) advertises a `duckdb` worker tag but **cannot** execute DuckDB scripts and has **no** working S3 proxy (DuckLake writes 404). Build CE DuckLake with `cargo run --features quickjs,duckdb,parquet,private` (add `,python` for Python scripts, `,enterprise,license` for EE) **and** build the FFI (bullet above). See `backend/CLAUDE.md` → "Running data pipelines (DuckLake) from source" for the exact feature sets and the two feature-gate gotchas. @@ -33,6 +41,37 @@ Open-source platform for internal tools, workflows, API integrations, background - **Instance settings**: navigate to `/#superadmin-settings` - **Migrations**: use `cargo sqlx migrate add -r ` from `backend/` to create new migrations (never generate timestamps manually) +### Per-worktree ports and database + +A worktree's `.env` / `.env.local` (repo root) and `backend/.env` hold its own +`DATABASE_URL` and `PORT`; the database is typically `windmill_` +(branch `dbt-runtime` → `windmill_dbt_runtime`). Read them, or discover from what is +already running: + +```bash +psql postgres://postgres:changeme@localhost:5432/postgres -tAc \ + "select datname from pg_database where datname like 'windmill%'" | grep "$(git branch --show-current | tr - _)" +# the port the frontend actually proxies to (REMOTE of this worktree's vite): +for p in $(pgrep -f vite); do case "$(readlink /proc/$p/cwd)" in *"$(basename "$(git rev-parse --show-toplevel)")"*) + tr '\0' '\n' < /proc/$p/environ | grep -E '^REMOTE=|^PORT=';; esac; done +``` + +Getting these wrong is not a cheap mistake: + +- **`DATABASE_URL` pointed at another worktree's database silently destroys the sqlx + cache.** `cargo run` and `cargo sqlx prepare` both compile `sqlx::query!` against the + **live** database, so the wrong one fails with `relation "" does not + exist` — and `prepare` deletes the whole `.sqlx/` directory *before* it fails, leaving + it gutted. Always `cp -r backend/.sqlx /sqlx_backup` first (see the `update-sqlx` + skill). +- **The frontend proxies to its own worktree's backend port, not 8000.** Starting a + backend on the wrong port leaves the UI up but every API call 502s, which reads like an + application bug rather than a misconfiguration. +- **Kill backends by pid scoped to this worktree's cwd** (`readlink /proc//cwd`), + never `pkill -f target/debug/windmill` — that kills every sibling worktree's backend. + Beware that a `pgrep -f ""` in a shell whose own command line contains + `` matches the shell itself. + ## Verifying Frontend Changes After modifying frontend code, drive the running dev server with the **Playwright MCP** to verify the change in a real browser — don't claim a UI change works without exercising it. @@ -55,6 +94,25 @@ Typical flow: If you cannot exercise a UI change (no dev server, etc.), say so explicitly rather than claiming success. +## Verifying Backend Changes + +`cargo check` and the unit tests do not exercise a worker code path. **If you changed how +a job runs — an executor, `handle_child`, anything spawning or reading from a +subprocess — run an actual job of that kind** and confirm it completed, then say so. +Whole classes of defect compile and unit-test clean: + +- **Stack overflow from a large buffer in an async block.** An array declared across an + `.await` is baked into the future's state; once that future is boxed a few layers deep + by the job poller, two 16 KB arrays abort the worker *process* (`thread + 'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers + (`vec![0u8; N]`, not `[0u8; N]`). +- Deadlocks from draining only one of a child's pipes, missed cancellation or timeout + propagation, and anything depending on the real engine's output format. + +A crash like this takes down every job on that worker, not just yours, so check the +backend log after the run rather than only the job's own status. If you cannot run one, +say which path went unexercised instead of implying it was verified. + ## Banned Patterns ### `$bindable(default_value)` on optional props diff --git a/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json b/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json index a0d8b66a26..d2828a36a4 100644 --- a/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json +++ b/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5.json b/backend/.sqlx/query-03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5.json new file mode 100644 index 0000000000..f99e53856c --- /dev/null +++ b/backend/.sqlx/query-03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest)\n VALUES ($1, $2, $3, $4, 'd2')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5" +} diff --git a/backend/.sqlx/query-04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480.json b/backend/.sqlx/query-04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480.json new file mode 100644 index 0000000000..4a632d868e --- /dev/null +++ b/backend/.sqlx/query-04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, runnable_path FROM v2_job\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480" +} diff --git a/backend/.sqlx/query-09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e.json b/backend/.sqlx/query-09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e.json index 60c3820b3d..8840ce4958 100644 --- a/backend/.sqlx/query-09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e.json +++ b/backend/.sqlx/query-09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e.json @@ -119,7 +119,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6.json b/backend/.sqlx/query-0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6.json new file mode 100644 index 0000000000..99bf8ee009 --- /dev/null +++ b/backend/.sqlx/query-0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_edge WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6" +} diff --git a/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json b/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json new file mode 100644 index 0000000000..e74bd5de0e --- /dev/null +++ b/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007" +} diff --git a/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json b/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json index 238522a3ff..62e65c847b 100644 --- a/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json +++ b/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } @@ -55,7 +56,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json b/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json index 4194e7e9cc..0bc0c3ce03 100644 --- a/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json +++ b/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json @@ -35,7 +35,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-11d979216951a85835c15e02c756249fa16fd913dc1a2c7babc329cc2d636342.json b/backend/.sqlx/query-11d979216951a85835c15e02c756249fa16fd913dc1a2c7babc329cc2d636342.json new file mode 100644 index 0000000000..4c3c53120c --- /dev/null +++ b/backend/.sqlx/query-11d979216951a85835c15e02c756249fa16fd913dc1a2c7babc329cc2d636342.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_graph_snapshot WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "11d979216951a85835c15e02c756249fa16fd913dc1a2c7babc329cc2d636342" +} diff --git a/backend/.sqlx/query-12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66.json b/backend/.sqlx/query-12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66.json new file mode 100644 index 0000000000..a797527421 --- /dev/null +++ b/backend/.sqlx/query-12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest)\n VALUES ($1, $2, $3, $4, 'd')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66" +} diff --git a/backend/.sqlx/query-19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da.json b/backend/.sqlx/query-19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da.json new file mode 100644 index 0000000000..76404ffce5 --- /dev/null +++ b/backend/.sqlx/query-19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da" +} diff --git a/backend/.sqlx/query-1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27.json b/backend/.sqlx/query-1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27.json new file mode 100644 index 0000000000..7f1618b3ad --- /dev/null +++ b/backend/.sqlx/query-1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_progress\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND asset_path = ANY($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27" +} diff --git a/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json b/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json index 9149698131..84ce6720d6 100644 --- a/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json +++ b/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e.json b/backend/.sqlx/query-1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e.json new file mode 100644 index 0000000000..57861d6518 --- /dev/null +++ b/backend/.sqlx/query-1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e" +} diff --git a/backend/.sqlx/query-2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0.json b/backend/.sqlx/query-2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0.json deleted file mode 100644 index 14f17d871e..0000000000 --- a/backend/.sqlx/query-2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_settings\n SET\n ai_config = source_ws.ai_config,\n large_file_storage = source_ws.large_file_storage,\n ducklake = source_ws.ducklake,\n datatable = source_ws.datatable,\n git_app_installations = source_ws.git_app_installations\n FROM workspace_settings source_ws\n WHERE source_ws.workspace_id = $1\n AND workspace_settings.workspace_id = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0" -} diff --git a/backend/.sqlx/query-229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07.json b/backend/.sqlx/query-229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07.json new file mode 100644 index 0000000000..1e92b86bb2 --- /dev/null +++ b/backend/.sqlx/query-229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags, description,\n columns, freshness)\n VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders',\n 'u/a/wh/analytics/orders', 'select 1', '{finance}', 'daily order facts',\n '{\"order_id\": {\"description\": \"natural key\"}}'::jsonb,\n '{\"warn_after\": {\"count\": 12, \"period\": \"hour\"}}'::jsonb)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07" +} diff --git a/backend/.sqlx/query-22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049.json b/backend/.sqlx/query-22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049.json new file mode 100644 index 0000000000..baef76a508 --- /dev/null +++ b/backend/.sqlx/query-22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH keep AS (\n SELECT hash FROM script\n WHERE workspace_id = $2 AND path = $3 AND language = 'dbt'\n ORDER BY created_at DESC LIMIT $1\n )\n DELETE FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $2 AND g.script_path = $3\n AND g.job_id = '00000000-0000-0000-0000-000000000000'\n AND NOT EXISTS (SELECT 1 FROM keep k WHERE k.hash = g.script_hash)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049" +} diff --git a/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json b/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json index e3f72d9c3a..ff79b3bfa5 100644 --- a/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json +++ b/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json @@ -27,7 +27,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json index 02a419de06..73feccaf09 100644 --- a/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json +++ b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json @@ -21,7 +21,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json b/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json index f940f25b4a..b2c5b915f5 100644 --- a/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json +++ b/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json @@ -42,7 +42,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680.json b/backend/.sqlx/query-2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680.json new file mode 100644 index 0000000000..f770ee58df --- /dev/null +++ b/backend/.sqlx/query-2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT relation_root_at_last_ingest FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3\n AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "relation_root_at_last_ingest", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + true + ] + }, + "hash": "2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680" +} diff --git a/backend/.sqlx/query-2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3.json b/backend/.sqlx/query-2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3.json new file mode 100644 index 0000000000..558e5e466e --- /dev/null +++ b/backend/.sqlx/query-2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3 AND retryable", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3" +} diff --git a/backend/.sqlx/query-2f883466fb4237a3bee51813d1b5cc16f6216de6e483f3bbb1c2f54b51a37a24.json b/backend/.sqlx/query-2f883466fb4237a3bee51813d1b5cc16f6216de6e483f3bbb1c2f54b51a37a24.json new file mode 100644 index 0000000000..a0afad1b40 --- /dev/null +++ b/backend/.sqlx/query-2f883466fb4237a3bee51813d1b5cc16f6216de6e483f3bbb1c2f54b51a37a24.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot\n (workspace_id, script_path, script_hash, job_id, digest, ingested_at)\n VALUES ($1, $2, $3, $4, $5, now())\n ON CONFLICT (workspace_id, script_path, script_hash, job_id)\n DO UPDATE SET digest = EXCLUDED.digest, ingested_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2f883466fb4237a3bee51813d1b5cc16f6216de6e483f3bbb1c2f54b51a37a24" +} diff --git a/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json index ded7b65a83..832a118e11 100644 --- a/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json +++ b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json @@ -21,7 +21,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28.json b/backend/.sqlx/query-33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28.json new file mode 100644 index 0000000000..8316ec5fab --- /dev/null +++ b/backend/.sqlx/query-33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_node WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28" +} diff --git a/backend/.sqlx/query-38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df.json b/backend/.sqlx/query-38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df.json new file mode 100644 index 0000000000..a779a3c995 --- /dev/null +++ b/backend/.sqlx/query-38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_node WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df" +} diff --git a/backend/.sqlx/query-381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549.json b/backend/.sqlx/query-381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549.json new file mode 100644 index 0000000000..18df498bba --- /dev/null +++ b/backend/.sqlx/query-381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET archived = true, extra_perms = '{\"u/outsider\": true}'::jsonb\n WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549" +} diff --git a/backend/.sqlx/query-3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d.json b/backend/.sqlx/query-3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d.json new file mode 100644 index 0000000000..d8afb95017 --- /dev/null +++ b/backend/.sqlx/query-3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders',\n 'u/a/wh/analytics/orders', 'select 2', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d" +} diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index 257e1e528f..2a02e5a8ce 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -77,7 +77,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b.json b/backend/.sqlx/query-41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b.json new file mode 100644 index 0000000000..c812a91912 --- /dev/null +++ b/backend/.sqlx/query-41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_progress\n WHERE workspace_id = $1 AND updated_at < now() - make_interval(days => $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b" +} diff --git a/backend/.sqlx/query-4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c.json b/backend/.sqlx/query-4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c.json new file mode 100644 index 0000000000..4762ed3279 --- /dev/null +++ b/backend/.sqlx/query-4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_state SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c" +} diff --git a/backend/.sqlx/query-4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e.json b/backend/.sqlx/query-4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e.json new file mode 100644 index 0000000000..a2e8f4127c --- /dev/null +++ b/backend/.sqlx/query-4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content,\n created_by, language, lock)\n VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e" +} diff --git a/backend/.sqlx/query-44ee3944c35080999850a85857d6e5ce78c738c163b3ccdc3fe252b2980f6f1d.json b/backend/.sqlx/query-44ee3944c35080999850a85857d6e5ce78c738c163b3ccdc3fe252b2980f6f1d.json new file mode 100644 index 0000000000..e3d6f0be65 --- /dev/null +++ b/backend/.sqlx/query-44ee3944c35080999850a85857d6e5ce78c738c163b3ccdc3fe252b2980f6f1d.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, runnable_path,\n CASE WHEN kind = 'script' THEN runnable_id END AS script_hash\n FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + true, + null + ] + }, + "hash": "44ee3944c35080999850a85857d6e5ce78c738c163b3ccdc3fe252b2980f6f1d" +} diff --git a/backend/.sqlx/query-471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339.json b/backend/.sqlx/query-471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339.json new file mode 100644 index 0000000000..5903661c1a --- /dev/null +++ b/backend/.sqlx/query-471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_id FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339" +} diff --git a/backend/.sqlx/query-493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91.json b/backend/.sqlx/query-493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91.json new file mode 100644 index 0000000000..04626fefe3 --- /dev/null +++ b/backend/.sqlx/query-493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET dbt_warehouses = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91" +} diff --git a/backend/.sqlx/query-4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61.json b/backend/.sqlx/query-4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61.json new file mode 100644 index 0000000000..a1cd77128d --- /dev/null +++ b/backend/.sqlx/query-4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT dbt_warehouses FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "dbt_warehouses", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [ + true + ] + }, + "hash": "4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61" +} diff --git a/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json b/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json index a7f5d2f785..e4e2c240c7 100644 --- a/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json +++ b/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json @@ -71,7 +71,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json b/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json index 6a010dd1f4..f1941c012c 100644 --- a/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json +++ b/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json @@ -35,7 +35,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json b/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json index c617944b91..b00c398445 100644 --- a/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json +++ b/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json @@ -41,7 +41,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8.json b/backend/.sqlx/query-56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8.json new file mode 100644 index 0000000000..134960fb28 --- /dev/null +++ b/backend/.sqlx/query-56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, child_unique_id, ingested_at)\n SELECT $2, script_path, script_hash, job_id, parent_unique_id, child_unique_id,\n ingested_at\n FROM dbt_edge\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8" +} diff --git a/backend/.sqlx/query-58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a.json b/backend/.sqlx/query-58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a.json new file mode 100644 index 0000000000..1edc69b002 --- /dev/null +++ b/backend/.sqlx/query-58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT result->'materialized' FROM v2_job_completed WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a" +} diff --git a/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json index c126b6371a..32284da567 100644 --- a/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json +++ b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } @@ -71,7 +72,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json index 03de8472ad..b7191f389c 100644 --- a/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json +++ b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json @@ -21,7 +21,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-5cda335f22d96fb8bf351f03248f669cfde49cc9872228998ce4a00b829250a1.json b/backend/.sqlx/query-5cda335f22d96fb8bf351f03248f669cfde49cc9872228998ce4a00b829250a1.json new file mode 100644 index 0000000000..7b172b5a63 --- /dev/null +++ b/backend/.sqlx/query-5cda335f22d96fb8bf351f03248f669cfde49cc9872228998ce4a00b829250a1.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.runnable_path, j.runnable_id\n FROM v2_job_queue q JOIN v2_job j ON j.id = q.id\n WHERE q.id = $1 AND j.workspace_id = $2 AND q.tag = ANY($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "5cda335f22d96fb8bf351f03248f669cfde49cc9872228998ce4a00b829250a1" +} diff --git a/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json b/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json index 8ff04d2fc1..34f1870b7a 100644 --- a/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json +++ b/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523.json b/backend/.sqlx/query-5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523.json new file mode 100644 index 0000000000..38d40a97c9 --- /dev/null +++ b/backend/.sqlx/query-5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT script_lang = 'dbt' FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Varchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523" +} diff --git a/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json b/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json index 80f172aa40..f6e000ba6f 100644 --- a/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json +++ b/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json @@ -45,7 +45,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json index 144bcb13f2..8d8ebfbed1 100644 --- a/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json +++ b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json @@ -22,7 +22,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e.json b/backend/.sqlx/query-6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e.json new file mode 100644 index 0000000000..1e99437ab0 --- /dev/null +++ b/backend/.sqlx/query-6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT asset_kind AS \"asset_kind: windmill_common::assets::AssetKind\", asset_path,\n status::text AS \"status!\", row_count, error\n FROM dbt_run_progress\n WHERE workspace_id = $1 AND job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume", + "table" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "row_count", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false, + null, + true, + true + ] + }, + "hash": "6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e" +} diff --git a/backend/.sqlx/query-71462a33d126a7b70c5ef570c63502cc7e848b82cc8a3e6032d8cd030ee8a1db.json b/backend/.sqlx/query-71462a33d126a7b70c5ef570c63502cc7e848b82cc8a3e6032d8cd030ee8a1db.json new file mode 100644 index 0000000000..7b1509ce02 --- /dev/null +++ b/backend/.sqlx/query-71462a33d126a7b70c5ef570c63502cc7e848b82cc8a3e6032d8cd030ee8a1db.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_node WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "71462a33d126a7b70c5ef570c63502cc7e848b82cc8a3e6032d8cd030ee8a1db" +} diff --git a/backend/.sqlx/query-717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0.json b/backend/.sqlx/query-717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0.json new file mode 100644 index 0000000000..6dc8e9acc2 --- /dev/null +++ b/backend/.sqlx/query-717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_node WHERE workspace_id = $1 AND script_hash = $2 AND job_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0" +} diff --git a/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json b/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json index 96a68f827d..64c6611843 100644 --- a/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json +++ b/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json @@ -84,7 +84,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f.json b/backend/.sqlx/query-71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f.json new file mode 100644 index 0000000000..397b310807 --- /dev/null +++ b/backend/.sqlx/query-71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE materialized_partition\n SET status = 'failed',\n error = COALESCE(error, 'the run ended before this model finished')\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND NOT (asset_path = ANY($3))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f" +} diff --git a/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json b/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json index b38e22ad67..d26e9f7c87 100644 --- a/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json +++ b/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json @@ -77,7 +77,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce.json b/backend/.sqlx/query-7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce.json new file mode 100644 index 0000000000..936a85b611 --- /dev/null +++ b/backend/.sqlx/query-7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT identity, args, run_results, job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "identity", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "args", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce" +} diff --git a/backend/.sqlx/query-7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77.json b/backend/.sqlx/query-7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77.json new file mode 100644 index 0000000000..ee215b9615 --- /dev/null +++ b/backend/.sqlx/query-7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_graph_snapshot WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77" +} diff --git a/backend/.sqlx/query-8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78.json b/backend/.sqlx/query-8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78.json deleted file mode 100644 index 5453ec6899..0000000000 --- a/backend/.sqlx/query-8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78" -} diff --git a/backend/.sqlx/query-843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc.json b/backend/.sqlx/query-843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc.json new file mode 100644 index 0000000000..5c000ffd34 --- /dev/null +++ b/backend/.sqlx/query-843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_graph_snapshot WHERE workspace_id = $1 AND job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc" +} diff --git a/backend/.sqlx/query-850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f.json b/backend/.sqlx/query-850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f.json new file mode 100644 index 0000000000..4c72eca991 --- /dev/null +++ b/backend/.sqlx/query-850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_graph_snapshot SET relation_root_at_last_ingest = $4\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3\n AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f" +} diff --git a/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json index 5aacf1a295..ed77b9e649 100644 --- a/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json +++ b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json b/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json index ffce4491e3..f26bca1232 100644 --- a/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json +++ b/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166.json b/backend/.sqlx/query-8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166.json new file mode 100644 index 0000000000..0f743221dc --- /dev/null +++ b/backend/.sqlx/query-8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language, lock)\n VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166" +} diff --git a/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json b/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json index 6f7468033a..44e6cc5c81 100644 --- a/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json +++ b/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json @@ -42,7 +42,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280.json b/backend/.sqlx/query-8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280.json new file mode 100644 index 0000000000..6f4d10b1f5 --- /dev/null +++ b/backend/.sqlx/query-8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_progress SET workspace_id = $1\n WHERE workspace_id = $2 AND job_id IN (SELECT id FROM v2_job WHERE workspace_id = $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280" +} diff --git a/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json index 24d70efae1..9e4209a706 100644 --- a/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json +++ b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json @@ -36,7 +36,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e.json b/backend/.sqlx/query-8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e.json new file mode 100644 index 0000000000..04d2fe0a77 --- /dev/null +++ b/backend/.sqlx/query-8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_graph_snapshot WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e" +} diff --git a/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json b/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json index eac4547361..3ea7be5cb8 100644 --- a/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json +++ b/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3.json b/backend/.sqlx/query-932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3.json new file mode 100644 index 0000000000..e2e3f374a6 --- /dev/null +++ b/backend/.sqlx/query-932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, on_behalf_of_email FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND deleted = false\n ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "on_behalf_of_email", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3" +} diff --git a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json new file mode 100644 index 0000000000..eb0aad5124 --- /dev/null +++ b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47" +} diff --git a/backend/.sqlx/query-96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9.json b/backend/.sqlx/query-96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9.json new file mode 100644 index 0000000000..5061c42c94 --- /dev/null +++ b/backend/.sqlx/query-96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM materialized_partition\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND asset_path = ANY($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9" +} diff --git a/backend/.sqlx/query-972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0.json b/backend/.sqlx/query-972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0.json new file mode 100644 index 0000000000..ec34b5f46a --- /dev/null +++ b/backend/.sqlx/query-972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2\n AND NOT EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0" +} diff --git a/backend/.sqlx/query-9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294.json b/backend/.sqlx/query-9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294.json new file mode 100644 index 0000000000..42b8a2b09c --- /dev/null +++ b/backend/.sqlx/query-9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, tags, test_kind, test_column, test_args,\n attached_node)\n VALUES ($1, $2, $3, $4, 'test.p.accepted_values_orders_status', 'test',\n 'accepted_values_orders_status', '{}', 'accepted_values', 'status',\n '{\"values\": [\"gold\", \"silver\"]}'::jsonb, 'model.p.orders')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294" +} diff --git a/backend/.sqlx/query-9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f.json b/backend/.sqlx/query-9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f.json index 1b11e0c470..b73ee34566 100644 --- a/backend/.sqlx/query-9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f.json +++ b/backend/.sqlx/query-9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f.json @@ -124,7 +124,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-9d97c18abb292c841c3c95510022b6abc0829349a2b189083f61cf06c15113fe.json b/backend/.sqlx/query-9d97c18abb292c841c3c95510022b6abc0829349a2b189083f61cf06c15113fe.json new file mode 100644 index 0000000000..493653e3dd --- /dev/null +++ b/backend/.sqlx/query-9d97c18abb292c841c3c95510022b6abc0829349a2b189083f61cf06c15113fe.json @@ -0,0 +1,134 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n JOIN live l ON l.path = n.script_path AND l.hash = n.script_hash\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned version's nodes are the scope.\n AND ($3::bigint IS NOT NULL OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n ) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path AND l.hash = n.script_hash\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "unique_id!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "resource_type!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "name!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "asset_path", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "materialized", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "materialize_strategy", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "tags!", + "type_info": "TextArray" + }, + { + "ordinal": 8, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "test_kind", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "test_column", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "test_args", + "type_info": "Jsonb" + }, + { + "ordinal": 12, + "name": "severity", + "type_info": "Text" + }, + { + "ordinal": 13, + "name": "attached_node", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "columns", + "type_info": "Jsonb" + }, + { + "ordinal": 15, + "name": "freshness", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "raw_code", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "original_file_path", + "type_info": "Text" + }, + { + "ordinal": 18, + "name": "script_visible!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + null + ] + }, + "hash": "9d97c18abb292c841c3c95510022b6abc0829349a2b189083f61cf06c15113fe" +} diff --git a/backend/.sqlx/query-a2e7c8c0d1cc256e59976845c5189ccc84720581d7e3661c5eb2719c8ccff08a.json b/backend/.sqlx/query-a2e7c8c0d1cc256e59976845c5189ccc84720581d7e3661c5eb2719c8ccff08a.json new file mode 100644 index 0000000000..679e03a3c7 --- /dev/null +++ b/backend/.sqlx/query-a2e7c8c0d1cc256e59976845c5189ccc84720581d7e3661c5eb2719c8ccff08a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_edge WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a2e7c8c0d1cc256e59976845c5189ccc84720581d7e3661c5eb2719c8ccff08a" +} diff --git a/backend/.sqlx/query-a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41.json b/backend/.sqlx/query-a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41.json new file mode 100644 index 0000000000..de91a548ad --- /dev/null +++ b/backend/.sqlx/query-a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT envs FROM script WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "envs", + "type_info": "VarcharArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + true + ] + }, + "hash": "a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41" +} diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index 4824bba526..2b9ac8283b 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -134,7 +134,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json b/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json new file mode 100644 index 0000000000..0532a5d3a0 --- /dev/null +++ b/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET\n ai_config = source_ws.ai_config,\n large_file_storage = source_ws.large_file_storage,\n ducklake = source_ws.ducklake,\n dbt_warehouses = source_ws.dbt_warehouses,\n datatable = source_ws.datatable,\n git_app_installations = source_ws.git_app_installations\n FROM workspace_settings source_ws\n WHERE source_ws.workspace_id = $1\n AND workspace_settings.workspace_id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a" +} diff --git a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json index 0d1bb6cb41..0faa9d0e33 100644 --- a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json +++ b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json @@ -39,7 +39,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-a99c9c63fee048333bbf633bb92faaf06b89fb7396a1a7bf9f370c32d6ad67ee.json b/backend/.sqlx/query-a99c9c63fee048333bbf633bb92faaf06b89fb7396a1a7bf9f370c32d6ad67ee.json new file mode 100644 index 0000000000..14eb734864 --- /dev/null +++ b/backend/.sqlx/query-a99c9c63fee048333bbf633bb92faaf06b89fb7396a1a7bf9f370c32d6ad67ee.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT g.job_id FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $2\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a99c9c63fee048333bbf633bb92faaf06b89fb7396a1a7bf9f370c32d6ad67ee" +} diff --git a/backend/.sqlx/query-aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c.json b/backend/.sqlx/query-aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c.json new file mode 100644 index 0000000000..e417d1e948 --- /dev/null +++ b/backend/.sqlx/query-aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms)\n VALUES ($1, 'private', 'private', '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c" +} diff --git a/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json b/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json index 04131f8f88..6a370ef364 100644 --- a/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json +++ b/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9.json b/backend/.sqlx/query-ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9.json new file mode 100644 index 0000000000..52ebcb5422 --- /dev/null +++ b/backend/.sqlx/query-ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT hash FROM script WHERE workspace_id = $1 AND path = $2 AND deleted = false AND archived = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9" +} diff --git a/backend/.sqlx/query-af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7.json b/backend/.sqlx/query-af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7.json new file mode 100644 index 0000000000..0446f2ea8a --- /dev/null +++ b/backend/.sqlx/query-af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_state SET workspace_id = $1\n WHERE workspace_id = $2 AND job_id IN (SELECT id FROM v2_job WHERE workspace_id = $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7" +} diff --git a/backend/.sqlx/query-b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39.json b/backend/.sqlx/query-b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39.json new file mode 100644 index 0000000000..10adc6e68d --- /dev/null +++ b/backend/.sqlx/query-b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_edge WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39" +} diff --git a/backend/.sqlx/query-b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506.json b/backend/.sqlx/query-b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506.json new file mode 100644 index 0000000000..e29c1c0dbb --- /dev/null +++ b/backend/.sqlx/query-b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506" +} diff --git a/backend/.sqlx/query-b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4.json b/backend/.sqlx/query-b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4.json new file mode 100644 index 0000000000..b865865c1d --- /dev/null +++ b/backend/.sqlx/query-b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4" +} diff --git a/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json b/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json index eab78c7de8..9187cc29b7 100644 --- a/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json +++ b/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c.json b/backend/.sqlx/query-b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c.json new file mode 100644 index 0000000000..3aaa58da40 --- /dev/null +++ b/backend/.sqlx/query-b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_state SET script_path = $3 WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c" +} diff --git a/backend/.sqlx/query-bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472.json b/backend/.sqlx/query-bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472.json new file mode 100644 index 0000000000..09367d07d3 --- /dev/null +++ b/backend/.sqlx/query-bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_progress SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472" +} diff --git a/backend/.sqlx/query-bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167.json b/backend/.sqlx/query-bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167.json new file mode 100644 index 0000000000..501a8460a1 --- /dev/null +++ b/backend/.sqlx/query-bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167" +} diff --git a/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json b/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json index 2e194c9580..be50a16b17 100644 --- a/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json +++ b/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json @@ -41,7 +41,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index 809deb1a55..32e4c5404e 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -78,7 +78,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json b/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json index a051b068a5..48abc75347 100644 --- a/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json +++ b/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json @@ -34,7 +34,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3.json b/backend/.sqlx/query-c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3.json new file mode 100644 index 0000000000..5fb1a353e7 --- /dev/null +++ b/backend/.sqlx/query-c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_edge\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3" +} diff --git a/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json index 7981dbc983..2e08168c2f 100644 --- a/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json +++ b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } @@ -87,7 +88,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241.json b/backend/.sqlx/query-c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241.json new file mode 100644 index 0000000000..d8f9fbcee4 --- /dev/null +++ b/backend/.sqlx/query-c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_run_state (workspace_id, script_path, permissioned_as, identity, args, run_results, job_id, retryable, updated_at)\n SELECT $1::varchar, $2::varchar, $7::varchar, $3::text, $4::jsonb, $5::text, $6::uuid, $8::boolean, now()\n WHERE EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false\n AND language = 'dbt')\n ON CONFLICT (workspace_id, script_path, permissioned_as) DO UPDATE SET\n identity = EXCLUDED.identity, args = EXCLUDED.args,\n run_results = EXCLUDED.run_results, job_id = EXCLUDED.job_id,\n retryable = EXCLUDED.retryable, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Jsonb", + "Text", + "Uuid", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241" +} diff --git a/backend/.sqlx/query-c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558.json b/backend/.sqlx/query-c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558.json new file mode 100644 index 0000000000..8bc10809dc --- /dev/null +++ b/backend/.sqlx/query-c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT deleted FROM script WHERE workspace_id = $1 AND hash = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "deleted", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558" +} diff --git a/backend/.sqlx/query-d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b.json b/backend/.sqlx/query-d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b.json index 2578d91d10..0da783f056 100644 --- a/backend/.sqlx/query-d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b.json +++ b/backend/.sqlx/query-d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b.json @@ -149,7 +149,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json b/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json index c4526daccc..64b4056596 100644 --- a/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json +++ b/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json @@ -69,7 +69,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index 2660ca05ef..cd1e0c4c39 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -78,7 +78,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b.json b/backend/.sqlx/query-d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b.json new file mode 100644 index 0000000000..edd7f2f97a --- /dev/null +++ b/backend/.sqlx/query-d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_graph_snapshot\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b" +} diff --git a/backend/.sqlx/query-dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6.json b/backend/.sqlx/query-dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6.json new file mode 100644 index 0000000000..520d04c762 --- /dev/null +++ b/backend/.sqlx/query-dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_edge WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6" +} diff --git a/backend/.sqlx/query-e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165.json b/backend/.sqlx/query-e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165.json new file mode 100644 index 0000000000..36161cb99c --- /dev/null +++ b/backend/.sqlx/query-e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_node\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165" +} diff --git a/backend/.sqlx/query-e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609.json b/backend/.sqlx/query-e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609.json new file mode 100644 index 0000000000..4967ac0212 --- /dev/null +++ b/backend/.sqlx/query-e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_progress\n SET status = 'failed',\n error = COALESCE(error, 'the run ended before this model finished')\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND NOT (asset_path = ANY($3))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609" +} diff --git a/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json b/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json index 037c796fc7..175e2b51b6 100644 --- a/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json +++ b/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json @@ -17,7 +17,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "table" ] } } diff --git a/backend/.sqlx/query-e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7.json b/backend/.sqlx/query-e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7.json new file mode 100644 index 0000000000..9509a76604 --- /dev/null +++ b/backend/.sqlx/query-e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script_trigger\n WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2\n AND trigger_kind = 'asset' AND trigger_ref LIKE 'dbt://%'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7" +} diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index ad34773e36..29d3eede53 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -134,7 +134,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa.json b/backend/.sqlx/query-ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa.json new file mode 100644 index 0000000000..d6e1a1bb57 --- /dev/null +++ b/backend/.sqlx/query-ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_run_progress\n (workspace_id, job_id, asset_kind, asset_path, status, row_count, error, updated_at)\n VALUES ($1, $2, 'dbt', $3, $4, $5, $6, now())\n ON CONFLICT (workspace_id, job_id, asset_kind, asset_path)\n DO UPDATE SET status = EXCLUDED.status, row_count = EXCLUDED.row_count,\n error = EXCLUDED.error, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "Varchar", + { + "Custom": { + "name": "materialization_status", + "kind": { + "Enum": [ + "running", + "materialized", + "failed" + ] + } + } + }, + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa" +} diff --git a/backend/.sqlx/query-ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9.json b/backend/.sqlx/query-ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9.json new file mode 100644 index 0000000000..90e4ba1e36 --- /dev/null +++ b/backend/.sqlx/query-ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job_queue q JOIN v2_job j ON j.id = q.id\n WHERE q.id = $1 AND j.workspace_id = $2 AND q.tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9" +} diff --git a/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json b/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json index 33b0b1f144..2357865ebc 100644 --- a/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json +++ b/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-ee4bf2bbc5ee6a49d7b5493793bb6771e9a2bcc847c5b64ee784b469242a1717.json b/backend/.sqlx/query-ee4bf2bbc5ee6a49d7b5493793bb6771e9a2bcc847c5b64ee784b469242a1717.json new file mode 100644 index 0000000000..89a7be2f20 --- /dev/null +++ b/backend/.sqlx/query-ee4bf2bbc5ee6a49d7b5493793bb6771e9a2bcc847c5b64ee784b469242a1717.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH live AS (\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- \u2014 silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n )\n SELECT p.asset_path AS \"from_path!\", c.asset_path AS \"to_path!\"\n FROM dbt_edge e\n JOIN live l ON l.path = e.script_path AND l.hash = e.script_hash\n JOIN chosen ch ON ch.job_id = e.job_id\n JOIN dbt_node p ON p.workspace_id = e.workspace_id\n AND p.script_path = e.script_path\n AND p.script_hash = e.script_hash\n AND p.job_id = ch.job_id\n AND p.unique_id = e.parent_unique_id\n JOIN dbt_node c ON c.workspace_id = e.workspace_id\n AND c.script_path = e.script_path\n AND c.script_hash = e.script_hash\n AND c.job_id = ch.job_id\n AND c.unique_id = e.child_unique_id\n WHERE e.workspace_id = $1\n AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL\n -- Tests attach to their model as a badge, not as a lineage edge.\n AND c.resource_type <> 'test'\n -- Scoped by the RELATIONS, like the node provenance above, not by\n -- the producing script's folder: two tables consumed in this\n -- folder but produced by a dbt project outside it would otherwise\n -- both render with their `ref()` edge missing.\n -- Same as the node scope: pinned, `asset` describes the CURRENT\n -- deploy, so gating on it drops the edges of models this version\n -- had and a later one removed. The pinned version's own edges are\n -- the answer.\n AND ($3::bigint IS NOT NULL OR EXISTS (\n SELECT 1 FROM asset a\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.path = c.asset_path\n AND ($2::text IS NULL OR a.usage_path LIKE $2)))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "from_path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "to_path!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ee4bf2bbc5ee6a49d7b5493793bb6771e9a2bcc847c5b64ee784b469242a1717" +} diff --git a/backend/.sqlx/query-eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c.json b/backend/.sqlx/query-eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c.json new file mode 100644 index 0000000000..e02c5897a4 --- /dev/null +++ b/backend/.sqlx/query-eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id,\n digest, relation_root_at_last_ingest, ingested_at)\n SELECT $2, script_path, script_hash, job_id, digest, relation_root_at_last_ingest,\n ingested_at\n FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c" +} diff --git a/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json b/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json index ad6028c14f..1bd565d00c 100644 --- a/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json +++ b/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce.json b/backend/.sqlx/query-f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e.json similarity index 83% rename from backend/.sqlx/query-88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce.json rename to backend/.sqlx/query-f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e.json index 9097eb8584..81034a3861 100644 --- a/backend/.sqlx/query-88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce.json +++ b/backend/.sqlx/query-f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n ai_config,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n ai_config,\n dbt_warehouses,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", "describe": { "columns": [ { @@ -80,81 +80,86 @@ }, { "ordinal": 15, - "name": "large_file_storage", + "name": "dbt_warehouses", "type_info": "Jsonb" }, { "ordinal": 16, - "name": "datatable", + "name": "large_file_storage", "type_info": "Jsonb" }, { "ordinal": 17, - "name": "ducklake", + "name": "datatable", "type_info": "Jsonb" }, { "ordinal": 18, - "name": "git_sync", + "name": "ducklake", "type_info": "Jsonb" }, { "ordinal": 19, - "name": "deploy_ui", + "name": "git_sync", "type_info": "Jsonb" }, { "ordinal": 20, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 21, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 21, + "ordinal": 22, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 22, + "ordinal": 23, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 23, + "ordinal": 24, "name": "color", "type_info": "Varchar" }, { - "ordinal": 24, + "ordinal": 25, "name": "operator_settings", "type_info": "Jsonb" }, { - "ordinal": 25, + "ordinal": 26, "name": "git_app_installations", "type_info": "Jsonb" }, { - "ordinal": 26, + "ordinal": 27, "name": "auto_invite", "type_info": "Jsonb" }, { - "ordinal": 27, + "ordinal": 28, "name": "error_handler", "type_info": "Jsonb" }, { - "ordinal": 28, + "ordinal": 29, "name": "success_handler", "type_info": "Jsonb" }, { - "ordinal": 29, + "ordinal": 30, "name": "public_app_execution_limit_per_minute", "type_info": "Int4" }, { - "ordinal": 30, + "ordinal": 31, "name": "error_handler_fallback_to_instance_alerts", "type_info": "Bool" } @@ -190,6 +195,7 @@ true, true, true, + true, false, true, true, @@ -198,5 +204,5 @@ false ] }, - "hash": "88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce" + "hash": "f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e" } diff --git a/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json index 07191c20b8..197229bb4c 100644 --- a/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json +++ b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json @@ -41,7 +41,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json b/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json index 12eceb0110..57d80d6aaa 100644 --- a/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json +++ b/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json @@ -39,7 +39,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } @@ -86,7 +87,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194.json b/backend/.sqlx/query-f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194.json new file mode 100644 index 0000000000..8cae250396 --- /dev/null +++ b/backend/.sqlx/query-f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194" +} diff --git a/backend/.sqlx/query-f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8.json b/backend/.sqlx/query-f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8.json new file mode 100644 index 0000000000..51c665b245 --- /dev/null +++ b/backend/.sqlx/query-f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT asset_kind AS \"asset_kind: windmill_common::assets::AssetKind\", asset_path,\n status::text AS \"status!\", row_count, error\n FROM materialized_partition\n WHERE workspace_id = $1 AND job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume", + "table" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "row_count", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false, + null, + true, + true + ] + }, + "hash": "f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8" +} diff --git a/backend/.sqlx/query-fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179.json b/backend/.sqlx/query-fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179.json new file mode 100644 index 0000000000..d74c17a1ed --- /dev/null +++ b/backend/.sqlx/query-fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179" +} diff --git a/backend/.sqlx/query-ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2.json b/backend/.sqlx/query-ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2.json new file mode 100644 index 0000000000..13ed472958 --- /dev/null +++ b/backend/.sqlx/query-ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT digest FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3\n AND job_id = $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "digest", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index bd12841ccc..4f24929695 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14789,7 +14789,9 @@ dependencies = [ "serde", "serde_json", "sqlx", + "tokio", "tracing", + "uuid", "windmill-api-auth", "windmill-common", "windmill-parser-sql-asset", @@ -15854,8 +15856,11 @@ name = "windmill-parser-yaml" version = "1.775.2" dependencies = [ "anyhow", + "lazy_static", + "regex", "serde", "serde_json", + "serde_yml", "windmill-parser", "yaml-rust", ] @@ -16434,6 +16439,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "serde", "serde_json", + "serde_yml", "sha2 0.10.9", "sqlx", "tar", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 108c17635c..df1600e68b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -564ad8932e1488dfc4b2694d9b4e50c580d1b8ed +cfd98fe9f640d86f34a363519e09a1ee0a5dedb3 diff --git a/backend/migrations/20260725084224_add_dbt_lang.down.sql b/backend/migrations/20260725084224_add_dbt_lang.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20260725084224_add_dbt_lang.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20260725084224_add_dbt_lang.up.sql b/backend/migrations/20260725084224_add_dbt_lang.up.sql new file mode 100644 index 0000000000..ec99678460 --- /dev/null +++ b/backend/migrations/20260725084224_add_dbt_lang.up.sql @@ -0,0 +1,2 @@ +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'dbt'; +UPDATE config SET config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["dbt"]'::jsonb) WHERE name = 'worker__default' AND config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java", "duckdb", "ruby", "rlang"]}'::jsonb AND NOT config->'worker_tags' @> '"dbt"'::jsonb; diff --git a/backend/migrations/20260725084314_dbt_runtime.down.sql b/backend/migrations/20260725084314_dbt_runtime.down.sql new file mode 100644 index 0000000000..93fd7ef4b5 --- /dev/null +++ b/backend/migrations/20260725084314_dbt_runtime.down.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS dbt_run_progress; +DROP TABLE IF EXISTS dbt_run_state; +DROP TABLE IF EXISTS dbt_graph_snapshot; +DROP TABLE IF EXISTS dbt_edge; +DROP TABLE IF EXISTS dbt_node; + +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS dbt_warehouses; diff --git a/backend/migrations/20260725084314_dbt_runtime.up.sql b/backend/migrations/20260725084314_dbt_runtime.up.sql new file mode 100644 index 0000000000..c40441933d --- /dev/null +++ b/backend/migrations/20260725084314_dbt_runtime.up.sql @@ -0,0 +1,218 @@ +-- The dbt runtime's tables: the parsed manifest that becomes the asset graph, +-- the retry state a `dbt retry` resumes from, and a run's live per-model +-- progress. + +-- Physical-relation asset kind. Identity is the relation itself +-- (`//`, the warehouse named as the workspace +-- configures it), never the producing tool, so a dbt mart and a native script +-- reading the same table resolve to one node and the lineage is one graph +-- across the boundary (docs/dbt-runtime.md, decision 11). +ALTER TYPE ASSET_KIND ADD VALUE IF NOT EXISTS 'dbt'; + +-- Warehouses configured once for the workspace, so a dbt project needs no +-- connection knowledge to run: the descriptor names one by NAME (or nothing, for +-- `main`) and the value here points at the RESOURCE that holds the credentials, +-- exactly as `large_file_storage` does for buckets. A flat map keyed by name, +-- which is also what asset identity keys on. +-- +-- {"main": {"resource_path": "$res:u/admin/wh", "target": "prod"}, +-- "eu": {"resource_path": "$res:u/admin/wh_eu"}} +ALTER TABLE workspace_settings ADD COLUMN IF NOT EXISTS dbt_warehouses JSONB; + +-- Parsed dbt manifest, one row per dbt node. The full manifest.json is not +-- stored -- only the fields the asset graph renders. +-- +-- Keyed by (path, version, job). The VERSION because each deployed version of a +-- script keeps its own graph, so a finished run can be shown the project as it +-- was rather than whatever is deployed today. The JOB because a dynamic +-- descriptor -- a `{{ }}` placeholder in `vars` -- can resolve to a different +-- set of models per run, and those runs re-ingest; the zero UUID means "the +-- version's own graph, as deployed", which is what a static descriptor writes +-- once and every one of its runs reads. A sentinel rather than NULL because +-- `job_id` is part of the key and Postgres does not treat two NULLs as the same +-- key, so each re-ingest would add a row set instead of replacing one. +CREATE TABLE IF NOT EXISTS dbt_node ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT NOT NULL, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + -- dbt's own node id, e.g. `model.jaffle_shop.customers`. Stable across runs + -- and the join key for dbt_edge and for run_results ingestion. + unique_id TEXT NOT NULL, + resource_type TEXT NOT NULL, + name TEXT NOT NULL, + -- The physical relation this node produces, spelled as the matching `asset` + -- row's path (`//`). NULL for nodes with no + -- relation: ephemeral models, tests. + asset_path TEXT, + materialized TEXT, + -- Windmill's equivalent write strategy (replace | append | merge | scd2), + -- NULL when dbt's materialization has no analogue (view, ephemeral). + materialize_strategy TEXT, + unique_key TEXT, + tags TEXT[] NOT NULL DEFAULT '{}', + description TEXT, + -- Test nodes only: the generic test name (`unique`, `not_null`, + -- `accepted_values`, `relationships`, or a package/custom test's own name), + -- the column it is attached to, its rendered kwargs and dbt's severity. + -- Severity is stored as dbt spells it; readers compare case-insensitively + -- because dbt-core 1.x echoes the author's casing while 2.x uppercases it. + test_kind TEXT, + test_column TEXT, + test_args JSONB, + severity TEXT, + attached_node TEXT, + -- Declared per-column metadata (name -> description), the column sets the + -- asset graph shows. dbt's manifest carries no column-to-column lineage, so + -- this is not a column lineage graph. + columns JSONB, + freshness JSONB, + -- The model's SQL as written, for the graph to render. `dbt parse` fills it; + -- `compiled_code` would need a `dbt compile`, which no phase runs. + raw_code TEXT, + -- Its path inside the dbt project, e.g. `models/staging/stg_orders.sql`. + original_file_path TEXT, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, script_hash, job_id, unique_id), + -- Composite because `script`'s key is (workspace_id, hash). A version's graph + -- dies with the version and nothing has to sweep it. + CONSTRAINT dbt_node_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- `ref()` / `source()` lineage, straight from the manifest's parent_map. +CREATE TABLE IF NOT EXISTS dbt_edge ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT NOT NULL, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + parent_unique_id TEXT NOT NULL, + child_unique_id TEXT NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, script_hash, job_id, parent_unique_id, child_unique_id), + CONSTRAINT dbt_edge_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- One row per stored graph, so a snapshot's EXISTENCE does not depend on it +-- having any nodes: a dynamic run can disable every model, and that empty graph +-- has to be distinguishable from a run that stored nothing -- otherwise the run +-- page falls back to the deployed models and shows what the run never built. +-- +-- It is also where the content digest lives, once. A run whose digest matches +-- the version's writes no snapshot at all: marking a descriptor dynamic is +-- conservative (a `{{ }}` in `vars` says the arguments reach dbt, not that they +-- change which models exist), so the usual dynamic run resolves to exactly the +-- graph the deploy stored. +CREATE TABLE IF NOT EXISTS dbt_graph_snapshot ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT NOT NULL, + job_id UUID NOT NULL, + digest TEXT NOT NULL, + -- On the DEPLOYED row only: the relation root the last ingest to write this + -- row resolved. Written by every ingest that is not a run's own snapshot, + -- including one that publishes no ownership — a version that cannot claim the + -- path would otherwise record nothing and compare against a stale root. + -- + -- The drift check needs "where do the current usages point", and no other + -- row answers it: the deploy's own root goes stale the moment a run at a + -- moved profile republishes, and "the newest ingest" is wrong because a run + -- whose graph matches the deploy's stores nothing at all. + relation_root_at_last_ingest TEXT, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, script_hash, job_id), + -- Same cascade as the rows it stands for. A marker outliving its nodes is + -- read as a snapshot that has none, and its digest still answers the + -- suppression check. + CONSTRAINT dbt_graph_snapshot_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- What a `dbt retry` resumes from. Keyed by path, not by version: there is one +-- saved run per script, and `identity` -- the project digest, warehouse and +-- engine -- is what refuses a resume that no longer describes the same run. +CREATE TABLE IF NOT EXISTS dbt_run_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + -- Part of the key: a retry replaces the caller's arguments with the saved + -- ones, so state written by one principal must not be restorable by another + -- — that would hand them the literal `select` and `vars` of a run they were + -- never entitled to see. + permissioned_as VARCHAR(255) NOT NULL, + identity TEXT NOT NULL, + -- The invocation's job arguments. `dbt retry` reuses the original selection + -- and vars, so the graph refresh and the build must agree with them rather + -- than with the retry request's. + args JSONB NOT NULL DEFAULT '{}'::jsonb, + -- Text rather than jsonb: nothing queries inside it, and this is highly + -- repetitive JSON that TOAST's compression handles well (about nine to one). + run_results TEXT NOT NULL, + job_id UUID, + -- Whether those results hold a node `dbt retry` would rebuild. A run that + -- failed before building anything, or succeeded outright, is still saved — + -- restoring it is how a retry can say the run succeeded rather than that no + -- state exists — but nothing may offer it as resumable. + retryable BOOLEAN NOT NULL DEFAULT false, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, permissioned_as) +); + +-- Live per-model progress for one RUN. +-- +-- `materialized_partition` cannot answer this: its key is the relation, one row +-- per table, and its `job_id` names only the last writer. Two runs of one +-- project building the same models take that row from each other, so a progress +-- read filtered by job loses nodes and flickers between states. That table is +-- left as it is -- the current state of a relation is what the pipeline canvas +-- and fork defer read, and one row per relation is right for them. +CREATE TABLE IF NOT EXISTS dbt_run_progress ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + job_id UUID NOT NULL, + asset_kind ASSET_KIND NOT NULL, + asset_path VARCHAR(255) NOT NULL, + status MATERIALIZATION_STATUS NOT NULL, + row_count BIGINT, + error TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, job_id, asset_kind, asset_path) +); + +-- Resolve a physical relation back to the dbt node that produces it: the asset +-- graph renders from `asset` rows and needs the dbt provenance per node. +CREATE INDEX IF NOT EXISTS idx_dbt_node_asset_path + ON dbt_node (workspace_id, asset_path) WHERE asset_path IS NOT NULL; + +-- No foreign key from the per-run rows to `v2_job`: three were deliberately +-- dropped from it for the write amplification they cost on the hottest table in +-- the system. Growth is bounded by age instead and pruned by the dbt runs +-- themselves, so no background sweep has to learn about these tables -- hence +-- an index per age sweep, and one per "does this job have a snapshot" lookup. +CREATE INDEX IF NOT EXISTS idx_dbt_node_job ON dbt_node (job_id) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_node_run_age ON dbt_node (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_edge_run_age ON dbt_edge (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_graph_snapshot_job + ON dbt_graph_snapshot (workspace_id, job_id); +CREATE INDEX IF NOT EXISTS idx_dbt_graph_snapshot_age + ON dbt_graph_snapshot (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_run_progress_updated_at + ON dbt_run_progress (updated_at); + +-- All of these are written on a user_db transaction (SET LOCAL ROLE +-- windmill_user/windmill_admin), so they must be granted explicitly rather than +-- relying on ALTER DEFAULT PRIVILEGES, which only covers objects created by the +-- role that set them (see 20260720131744). +GRANT ALL ON dbt_node TO windmill_user; +GRANT ALL ON dbt_node TO windmill_admin; +GRANT ALL ON dbt_edge TO windmill_user; +GRANT ALL ON dbt_edge TO windmill_admin; +GRANT ALL ON dbt_graph_snapshot TO windmill_user; +GRANT ALL ON dbt_graph_snapshot TO windmill_admin; +GRANT ALL ON dbt_run_state TO windmill_user; +GRANT ALL ON dbt_run_state TO windmill_admin; +GRANT ALL ON dbt_run_progress TO windmill_user; +GRANT ALL ON dbt_run_progress TO windmill_admin; diff --git a/backend/migrations/20260727145314_materialized_partition_job_id_index.down.sql b/backend/migrations/20260727145314_materialized_partition_job_id_index.down.sql new file mode 100644 index 0000000000..9b33bff978 --- /dev/null +++ b/backend/migrations/20260727145314_materialized_partition_job_id_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_materialized_partition_job; diff --git a/backend/migrations/20260727145314_materialized_partition_job_id_index.up.sql b/backend/migrations/20260727145314_materialized_partition_job_id_index.up.sql new file mode 100644 index 0000000000..6bcdc246ae --- /dev/null +++ b/backend/migrations/20260727145314_materialized_partition_job_id_index.up.sql @@ -0,0 +1,9 @@ +-- `job_id` leads neither the primary key nor the asset/status index, both shaped +-- for the per-relation lookups that came first — but the closing sweep a dbt run +-- makes (`terminalize_running_relations`) reads and writes by it, settling the +-- models a cancelled or timed-out run left `running`. +-- Small table until a partitioned asset is backfilled, then one row per slice +-- per asset. +CREATE INDEX IF NOT EXISTS idx_materialized_partition_job + ON materialized_partition (workspace_id, job_id) + WHERE job_id IS NOT NULL; diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index b8a12fb476..9c70b93d64 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -273,7 +273,7 @@ impl AssetsFinder { Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { let path = parse_asset_syntax(&value, false) .map(|(_, p)| p) - .unwrap_or(&value); + .unwrap_or(std::borrow::Cow::Borrowed(value)); self.assets.push(ParseAssetsResult { kind, path: path.to_string(), diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 0e4e12eb89..7811768720 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6227,6 +6227,7 @@ dependencies = [ "jsonwebtoken", "lazy_static", "magic-crypt", + "memchr", "native-tls", "once_cell", "pep440_rs", @@ -6266,6 +6267,7 @@ dependencies = [ "windmill-macros", "windmill-parser", "windmill-parser-sql", + "windmill-parser-sql-asset", "windmill-parser-ts", "windmill-types", ] @@ -6573,8 +6575,11 @@ name = "windmill-parser-yaml" version = "1.775.2" dependencies = [ "anyhow", + "lazy_static", + "regex", "serde", "serde_json", + "serde_yml", "windmill-parser", "yaml-rust", ] diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index c843652e26..59e963d52c 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -164,6 +164,14 @@ pub fn parse_ansible(code: &str) -> String { wrap_sig(windmill_parser_yaml::parse_ansible_sig(code)) } +/// Shares the ansible parser's feature: both live in windmill-parser-yaml, and +/// a dbt descriptor is YAML too. +#[cfg(feature = "ansible-parser")] +#[wasm_bindgen] +pub fn parse_dbt(code: &str) -> String { + wrap_sig(windmill_parser_yaml::parse_dbt_sig(code)) +} + #[cfg(feature = "ansible-parser")] #[wasm_bindgen] pub fn parse_ansible_delegate(code: &str) -> String { diff --git a/backend/parsers/windmill-parser-yaml/Cargo.toml b/backend/parsers/windmill-parser-yaml/Cargo.toml index eb01ba1936..eb2f16fccc 100644 --- a/backend/parsers/windmill-parser-yaml/Cargo.toml +++ b/backend/parsers/windmill-parser-yaml/Cargo.toml @@ -14,3 +14,6 @@ windmill-parser.workspace = true anyhow.workspace = true serde_json.workspace = true serde.workspace = true +serde_yml.workspace = true +regex.workspace = true +lazy_static.workspace = true diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs new file mode 100644 index 0000000000..e9778fd3f2 --- /dev/null +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -0,0 +1,768 @@ +//! The `ScriptLang::Dbt` script artifact: a YAML descriptor holding a dbt +//! project's run configuration. The project itself is the script's module +//! bundle, so the descriptor names no source for it. +//! +//! Field names track dbt's and astronomer-cosmos's vocabulary so the mental +//! model ports without translation (docs/dbt-runtime.md, decision 22). +//! `select` / `exclude` / `selector` are passed to dbt **verbatim**: the +//! selector grammar is dbt's, and reimplementing it is a standing source of +//! divergence. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use windmill_parser::{Arg, MainArgSignature, Typ}; + +/// Which dbt to run. The shipped default is `dbt-core-1x` because it runs +/// today's projects untouched; `fusion` is never bundled and is fetched from +/// dbt Labs at runtime (docs/dbt-runtime.md, decision 1). +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "kebab-case")] +pub enum DbtEngine { + #[default] + #[serde(rename = "dbt-core-1x")] + DbtCore1x, + #[serde(rename = "dbt-core-2x")] + DbtCore2x, + Fusion, +} + +impl DbtEngine { + pub fn as_str(&self) -> &'static str { + match self { + DbtEngine::DbtCore1x => "dbt-core-1x", + DbtEngine::DbtCore2x => "dbt-core-2x", + DbtEngine::Fusion => "fusion", + } + } + + /// The level the machine-readable file log is written at. Separate from the + /// console log, which always stays human-readable at the default level and + /// is what reaches the job log. + pub fn progress_log_level(&self) -> &'static str { + match self { + DbtEngine::DbtCore1x => "info", + DbtEngine::DbtCore2x | DbtEngine::Fusion => "debug", + } + } + + /// Whether the engine writes per-node events to its JSON **file** log — the + /// only source of *live* per-model status. + /// + /// Not a claim that the others produce none: dbt-core 2.x and Fusion emit + /// the same events, on the console, and ignore `--log-format-file json` + /// though both accept it. Reading them would mean taking over the console + /// and re-rendering the job log. Flip this the moment either honours the + /// flag — nothing else has to change (docs/dbt-runtime.md, "Live per-model + /// progress"). + pub fn emits_node_events(&self) -> bool { + matches!(self, DbtEngine::DbtCore1x) + } +} + +/// How the warehouse connection is supplied. Both paths are supported +/// (decision 8): render `profiles.yml` from a Windmill resource, or keep the +/// project's own file and inject Windmill secrets as env vars for +/// `{{ env_var() }}`. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(deny_unknown_fields)] +pub struct DbtProfile { + /// A warehouse configured on the workspace, by name; omitted takes the + /// default one. This is the ONLY way a project names a warehouse — there is + /// no per-descriptor resource — so a dbt project carries no connection at + /// all, the same bargain `s3://` and `ducklake://` make with workspace + /// storage, and asset identity has exactly one spelling to key on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub warehouse: Option, + + /// dbt target name. It selects which output of the profile runs; the + /// `` component of asset identity comes from `warehouse` above. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target: Option, + /// Path (relative to `project`) of the project's own `profiles.yml`, used + /// instead of rendering one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profiles_yml: Option, + /// Target schema (BigQuery calls it the dataset). Required for adapters + /// whose Windmill resource carries none — a BigQuery resource is a raw + /// service-account JSON, which has no dataset in it. Overrides the + /// resource's own value where there is one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// dbt adapter (`postgres` | `snowflake` | `bigquery` | `databricks`), + /// spelled as dbt's own `type:`. Optional: the worker infers it from the + /// resource's shape, and this pins it when the inference is wrong or the + /// resource is a custom type. + #[serde(default, rename = "type", skip_serializing_if = "Option::is_none")] + pub adapter: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(rename_all = "snake_case")] +pub enum DbtTestBehavior { + /// `dbt build` — models and their tests interleaved, a model's tests gating + /// its children. dbt's own default and the only behavior that stops bad + /// data propagating mid-run. + #[default] + Build, + /// `dbt run` then `dbt test`. + AfterAll, + /// `dbt run` only. + None, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(deny_unknown_fields)] +pub struct DbtDescriptor { + #[serde(default)] + pub engine: Option, + #[serde(default)] + pub profile: DbtProfile, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub select: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, + /// A named selector from the project's `selectors.yml`, passed verbatim. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + #[serde(default)] + pub test_behavior: DbtTestBehavior, + /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are + /// all normal — so values keep their YAML type; only string leaves carry + /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing + /// everything to a string would make a `false` var truthy in Jinja. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub vars: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub threads: Option, + #[serde(default)] + pub full_refresh: bool, + /// Automatic in-job retry of the nodes a build failed on. + /// + /// dbt already confines a failure to its own subtree, and `dbt retry` + /// rebuilds exactly the failed and skipped set, so a transient warehouse + /// error costs those nodes rather than the whole project. Doing it inside + /// the same job is what keeps the state question out of it: the previous + /// attempt's `run_results.json` is still in the job directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_failed_nodes: Option, + /// Extra environment for the dbt process, for the project's own + /// `{{ env_var() }}` lookups and for engine flags such as + /// `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`. A `$var:` value is resolved to + /// that Windmill variable's value by the worker, so a password never has to + /// sit in the descriptor — which is versioned script content. + /// + /// This is the map to use for anything the GRAPH depends on — an + /// `env_var()` driving a schema, alias or `enabled` — because it applies at + /// deploy as well as at run. Script-level environment variables reach the + /// run only, so a graph parsed without them would disagree with what the + /// build writes. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, +} + +/// How many times, and how far apart, to retry a build's failed nodes. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct DbtNodeRetry { + /// Extra `dbt retry` attempts for the job, spent across whichever phases + /// fail — a model phase that uses them all leaves none for the tests. + pub attempts: u32, + /// Seconds between attempts. A transient warehouse error is usually a lock + /// or a restart, so a pause is the point. + #[serde(default)] + pub delay_seconds: u64, +} + +impl DbtNodeRetry { + /// Bounded because each attempt is a real dbt invocation inside a job that + /// already holds a worker slot, and the job's own deadline still applies. + pub const MAX_ATTEMPTS: u32 = 10; + + pub fn attempts(&self) -> u32 { + self.attempts.min(Self::MAX_ATTEMPTS) + } +} + +/// The dbt subcommands a run may ask for. Kept here so the signature and the +/// worker's validation cannot drift apart. +/// +/// An allowlist rather than a passthrough: the value becomes the dbt subcommand, +/// and running a script needs weaker permission than editing it, so an unchecked +/// arg would let a runner invoke `clean` or `source freshness` against the +/// descriptor's warehouse. +/// +/// `run` is absent because it covers models only: a project with seeds or +/// snapshots would build a subset of what its graph claims. Narrowing what a run +/// touches is `select`/`exclude`, which scope the graph too. Tests run as part +/// of `build`, or as the second phase of `test_behavior: after_all`. +/// +/// `show` writes nothing — it SELECTs from a model and returns rows. That is +/// only admissible because a dbt run no longer dispatches: while it did, any +/// successful job fired the script's whole deploy-time write set, so a command +/// that built none of them woke every consumer for relations nothing touched. +pub const DBT_COMMANDS: &[&str] = &["build", "retry", "show"]; + +/// Rows a `show` returns unless the run asks for fewer. dbt enforces it, so the +/// bound is not us splicing a `LIMIT` into someone's SQL. +pub const DBT_SHOW_DEFAULT_LIMIT: u32 = 100; + +/// Hard ceiling on that limit. The worker buffers `dbt show`'s whole stdout to +/// read the rows out of it, so the argument decides how much memory a caller can +/// make it hold — and running a script needs only run permission. A preview is +/// for looking at a few rows; anything larger is a query, which is what a SQL +/// script is for. +pub const DBT_SHOW_MAX_LIMIT: u32 = 1_000; + +/// Whether the command only reads. Such a run publishes no graph, records no +/// materializations and runs no test phase — there is nothing it could have +/// changed. +pub fn is_read_only_command(command: &str) -> bool { + command == "show" +} + +/// The command a run uses when it does not name one. Public because the worker +/// must choose exactly what the run form's default advertises. +pub fn default_command(d: &DbtDescriptor) -> &'static str { + match d.test_behavior { + DbtTestBehavior::Build => "build", + // Also `build`, with the tests excluded rather than the command + // narrowed to `run`: `run` covers models only, so a selection that + // includes a seed or a snapshot would silently not build it and the + // models reading it would fail — or worse, read a stale table. + DbtTestBehavior::AfterAll | DbtTestBehavior::None => "build", + } +} + +impl DbtDescriptor { + pub fn engine(&self) -> DbtEngine { + self.engine.unwrap_or_default() + } +} + +/// Names a `{{ placeholder }}` may not take. `command` is the argument holding +/// the run's command block; the rest are the fields inside it, which the worker +/// spreads over the run's arguments to read them — a placeholder of the same +/// name would be shadowed there, and the script could never be run (`select` is +/// an array, and interpolating one into a string is not something any invocation +/// can satisfy). +pub const RESERVED_ARG_NAMES: &[&str] = &[ + "command", + "select", + "exclude", + "vars", + "full_refresh", + "dbt_command", + "dbt_retry_job", + "model", + "limit", +]; + +pub fn parse_dbt_descriptor(inner_content: &str) -> anyhow::Result { + let d = serde_yml::from_str::(inner_content) + .map_err(|e| anyhow::anyhow!("Failed to parse dbt descriptor: {e}"))?; + if let Some(name) = placeholders(&d) + .into_iter() + .find(|n| RESERVED_ARG_NAMES.contains(&n.as_str())) + { + return Err(anyhow::anyhow!( + "`{{{{ {name} }}}}` collides with the run argument `{name}` this runtime already \ + defines ({}); rename the placeholder", + RESERVED_ARG_NAMES.join(", ") + )); + } + Ok(d) +} + +/// The workspace warehouse a descriptor gets when it names none — spelled like +/// the default lake (`ducklake://main.orders`), so one workspace concept reads +/// the same across kinds. +pub const DBT_DEFAULT_WAREHOUSE: &str = "main"; + +/// The single argument holding the command and the overrides it takes. Its +/// variant IS the command, so a run cannot carry an override the command +/// ignores: `retry` rebuilds the failed run's nodes with the arguments that run +/// had, and `full_refresh` means nothing to a `show`. +pub const DBT_COMMAND_ARG: &str = "command"; + +/// The variant discriminator, which is the key Windmill's run form tags a +/// `oneOf` value with. +pub const DBT_COMMAND_LABEL: &str = "label"; + +fn list_arg(name: &str, default: &[String]) -> Arg { + Arg { + name: name.to_string(), + otyp: None, + typ: Typ::List(Box::new(Typ::Str(None))), + has_default: true, + default: Some(serde_json::json!(default)), + oidx: None, + otyp_inferred: false, + } +} + +/// An OVERRIDE map, so its default is empty rather than a copy of the +/// descriptor's vars. Seeding it with the descriptor would make the run form +/// post the raw `{{ placeholder }}` text back and clobber the value the worker +/// just interpolated for it. +fn vars_arg() -> Arg { + Arg { + name: "vars".to_string(), + otyp: None, + typ: Typ::Object(windmill_parser::ObjectType::new(None, Some(vec![]))), + has_default: true, + default: Some(serde_json::json!({})), + oidx: None, + otyp_inferred: false, + } +} + +/// What each command takes, in form order. +/// +/// `show` is absent on purpose: it previews ONE model's rows, which is a thing +/// to do to a table you are looking at rather than a job to fill a form in for. +/// The run page's graph and the assets list offer it where the tables are; the +/// worker still accepts `{label: show, model, limit}` from a flow, the CLI or the +/// API (`DBT_COMMANDS`, docs/dbt-runtime.md). +fn command_variants(d: &DbtDescriptor) -> Vec<(&'static str, Vec)> { + let selection = || { + vec![ + list_arg("select", &d.select), + list_arg("exclude", &d.exclude), + vars_arg(), + ] + }; + vec![ + ( + "build", + selection() + .into_iter() + .chain([Arg { + name: "full_refresh".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.full_refresh)), + oidx: None, + otyp_inferred: false, + }]) + .collect(), + ), + ( + "retry", + // The run it resumes, named rather than implied: only the latest + // failure of this script is kept, so "resume the last one" would + // silently aim somewhere else the moment another run failed. No + // default, so this variant requires it. + vec![Arg { + name: "dbt_retry_job".to_string(), + otyp: None, + typ: Typ::Str(None), + has_default: false, + default: None, + oidx: None, + otyp_inferred: false, + }], + ), + ] +} + +/// The command block a run gets when it overrides nothing: the descriptor's own +/// selection under the descriptor's default command, so an untouched run +/// reproduces it exactly. +fn default_command_value(d: &DbtDescriptor) -> serde_json::Value { + let label = default_command(d); + let mut obj = serde_json::Map::new(); + obj.insert(DBT_COMMAND_LABEL.to_string(), serde_json::json!(label)); + for arg in command_variants(d) + .into_iter() + .find(|(l, _)| *l == label) + .map(|(_, args)| args) + .unwrap_or_default() + { + if let Some(default) = arg.default { + obj.insert(arg.name, default); + } + } + serde_json::Value::Object(obj) +} + +/// Run-time arguments of a dbt script: the command block above, plus one +/// argument per `{{ placeholder }}` the descriptor interpolates. Placeholders +/// stay top-level — they are the project's own inputs, not a command's. +pub fn parse_dbt_sig(inner_content: &str) -> anyhow::Result { + let d = parse_dbt_descriptor(inner_content)?; + let mut args = vec![Arg { + name: DBT_COMMAND_ARG.to_string(), + otyp: None, + typ: Typ::Object(windmill_parser::ObjectType::new(None, Some(vec![]))), + has_default: true, + default: Some(default_command_value(&d)), + oidx: None, + otyp_inferred: false, + }]; + + for name in placeholders(&d) { + if args.iter().any(|a| a.name == name) { + continue; + } + args.push(Arg { + name, + otyp: None, + // Untyped, not `Str`: a placeholder standing alone in a var takes + // the argument's own JSON type, and declaring it a string makes the + // run form post `"false"` — truthy in Jinja — for a boolean. + typ: Typ::Unknown, + has_default: false, + default: None, + oidx: None, + otyp_inferred: false, + }); + } + + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + auto_kind: None, + has_preprocessor: None, + ..Default::default() + }) +} + +/// The run form's JSON schema for a dbt descriptor. +/// +/// Derived here rather than in the browser or the CLI: both infer a script's +/// schema client-side through `windmill-parser-wasm`, whose published package +/// has no dbt arm, so they leave the schema untouched. Without this a dbt +/// script deploys with an empty schema and its run form offers none of the +/// overrides — and an edited descriptor keeps the previous one's arguments. +/// The command is a `oneOf` rather than an enum beside the overrides it selects: +/// a variant carries exactly the arguments its command takes, so `dbt_retry_job` +/// is required where it means something and absent everywhere else, and no run +/// can submit a `full_refresh` to a command that ignores it. The run form renders +/// it as a toggle over the variants and tags the value with `label`. +pub fn dbt_arg_schema(inner_content: &str) -> anyhow::Result { + let d = parse_dbt_descriptor(inner_content)?; + let sig = parse_dbt_sig(inner_content)?; + let mut properties = serde_json::Map::new(); + let mut required: Vec = vec![]; + let mut order: Vec = vec![]; + + let variants: Vec = command_variants(&d) + .into_iter() + .map(|(label, args)| { + let mut props = serde_json::Map::new(); + let mut var_required: Vec = vec![]; + let mut var_order: Vec = vec![serde_json::json!(DBT_COMMAND_LABEL)]; + // The discriminator. Single-valued, so the form's toggle is what sets + // it and a submitted block cannot claim one command while carrying + // another's fields. + props.insert( + DBT_COMMAND_LABEL.to_string(), + serde_json::json!({"type": "string", "enum": [label]}), + ); + for arg in args { + if !arg.has_default { + var_required.push(serde_json::json!(arg.name)); + } + var_order.push(serde_json::json!(arg.name)); + props.insert(arg.name.clone(), property_of(&arg)); + } + serde_json::json!({ + "title": label, + "type": "object", + "properties": props, + "order": var_order, + "required": var_required, + }) + }) + .collect(); + + for arg in &sig.args { + let mut prop = property_of(arg); + if arg.name == DBT_COMMAND_ARG { + if let Some(obj) = prop.as_object_mut() { + obj.insert( + "oneOf".to_string(), + serde_json::Value::Array(variants.clone()), + ); + obj.insert( + "description".to_string(), + serde_json::json!( + "`build` runs the project. `retry` resumes a failed run, rebuilding only \ + its failed and skipped nodes with the arguments it ran with." + ), + ); + } + } + if !arg.has_default { + required.push(serde_json::json!(arg.name)); + } + order.push(serde_json::json!(arg.name)); + properties.insert(arg.name.clone(), prop); + } + Ok(serde_json::json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": properties, + "required": required, + "order": order, + })) +} + +/// One argument as a JSON-schema property, with the description the run form +/// shows under its label. +fn property_of(arg: &Arg) -> serde_json::Value { + let mut prop = match &arg.typ { + Typ::Bool => serde_json::json!({"type": "boolean"}), + // `limit` is an integer to the worker, which clamps it; without this the + // generated clients and the run form offer no numeric control. + Typ::Int => serde_json::json!({"type": "integer"}), + Typ::List(_) => serde_json::json!({"type": "array", "items": {"type": "string"}}), + Typ::Object(_) => serde_json::json!({"type": "object"}), + Typ::Str(Some(variants)) => serde_json::json!({"type": "string", "enum": variants}), + Typ::Str(None) => serde_json::json!({"type": "string"}), + // A placeholder takes the JSON type of whatever is passed, so it is + // deliberately left untyped rather than guessed at. + _ => serde_json::json!({}), + }; + if let Some(obj) = prop.as_object_mut() { + if let Some(default) = arg.default.as_ref() { + obj.insert("default".to_string(), default.clone()); + } + let description = match arg.name.as_str() { + "dbt_retry_job" => Some( + "The failed run to resume, by run id. Its failed and skipped nodes are rebuilt \ + with the arguments it ran with. Resuming from that run's page fills this in.", + ), + "select" => Some( + "dbt selection syntax, e.g. `tag:nightly`, `stg_orders+`, \ + `config.materialized:incremental`. Empty runs the descriptor's own selection.", + ), + "exclude" => Some("Nodes to leave out of the selection above, same syntax."), + "vars" => Some( + "dbt `--vars`, merged over the descriptor's. A var that changes which models \ + exist makes this run store its own graph rather than the deployed one.", + ), + "full_refresh" => Some("Rebuild incremental models from scratch instead of appending."), + "model" => Some( + "The model to preview, by name — `stg_orders`, or `my_package.stg_orders` when \ + two packages share a name. Any dbt selector resolving to ONE node works.", + ), + "limit" => Some("Rows the preview returns."), + // A placeholder the descriptor interpolates: its meaning is the + // project's, so there is nothing generic to say about it. + _ => None, + }; + if let Some(d) = description { + obj.insert("description".to_string(), serde_json::json!(d)); + } + } + prop +} + +/// `{{ name }}` placeholders in the interpolated descriptor fields, in a stable +/// order. Must stay in sync with the fields the worker actually interpolates. +fn placeholders(d: &DbtDescriptor) -> Vec { + let mut out: Vec = vec![]; + let mut push_from = |s: &str| { + for caps in PLACEHOLDER_RE.captures_iter(s) { + let name = caps[1].to_string(); + if !out.contains(&name) { + out.push(name); + } + } + }; + for v in d.vars.values() { + for leaf in string_leaves(v) { + push_from(leaf); + } + } + out +} + +/// Every string inside a var's value, at any depth — the only places a +/// `{{ arg }}` placeholder can appear. +pub fn string_leaves(v: &serde_json::Value) -> Vec<&str> { + match v { + serde_json::Value::String(s) => vec![s.as_str()], + serde_json::Value::Array(a) => a.iter().flat_map(string_leaves).collect(), + serde_json::Value::Object(o) => o.values().flat_map(string_leaves).collect(), + _ => vec![], + } +} + +lazy_static::lazy_static! { + /// Same spelling as the Ansible executor's `interpolate_template`, which is + /// what actually performs the substitution at run time. + static ref PLACEHOLDER_RE: regex::Regex = + regex::Regex::new(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}").unwrap(); +} + +#[cfg(test)] +mod tests { + use super::*; + + const DESCRIPTOR: &str = r#" +engine: dbt-core-2x +profile: + warehouse: main + target: prod +select: ["tag:nightly+"] +test_behavior: after_all +vars: + run_date: "{{ day }}" + strict: false +threads: 8 +full_refresh: true +"#; + + // A descriptor drives warehouse writes, so a field it does not recognise is + // an error rather than a default. `selcet:` would otherwise leave `select` + // empty and build the entire project; a misspelled `target` would silently + // fall back to the profile's own default. + #[test] + fn an_unknown_descriptor_field_is_refused() { + let err = parse_dbt_descriptor("profile:\n warehouse: main\nselcet: [a]\n") + .unwrap_err() + .to_string(); + assert!(err.contains("selcet"), "{err}"); + let err = parse_dbt_descriptor("profile:\n warehouse: main\n targt: prod\n") + .unwrap_err() + .to_string(); + assert!(err.contains("targt"), "{err}"); + } + + // A placeholder that takes a run argument's name would be shadowed by it, + // leaving a descriptor no invocation can satisfy: `select` is an array and + // the interpolation needs a scalar. Refused at parse, so the deploy says so + // rather than the script becoming unrunnable after it. + #[test] + fn a_placeholder_may_not_take_a_run_argument_name() { + for name in RESERVED_ARG_NAMES { + let d = + format!("profile:\n warehouse: main\nvars:\n v: \"{{{{ {name} }}}}\"\n"); + let err = parse_dbt_descriptor(&d).unwrap_err().to_string(); + assert!(err.contains(name), "{name}: {err}"); + } + // A name of its own is fine. + assert!(parse_dbt_descriptor( + "profile:\n warehouse: main\nvars:\n v: \"{{ day }}\"\n" + ) + .is_ok()); + } + + // The run form is built from this schema, and nothing else can build it: + // the browser and the CLI both infer through a wasm parser that has no dbt + // arm, so a missing property here is an override the user cannot reach. + // The variants are what make the form show a command's own fields and only + // those, so each is asserted by the arguments it carries. + #[test] + fn the_schema_carries_every_run_override_and_placeholder() { + let schema = dbt_arg_schema(DESCRIPTOR).unwrap(); + let props = schema["properties"].as_object().unwrap(); + let variants = props[DBT_COMMAND_ARG]["oneOf"].as_array().unwrap(); + let of = |label: &str| { + let v = variants + .iter() + .find(|v| v["title"] == label) + .unwrap_or_else(|| panic!("no `{label}` variant: {schema}")); + let mut names: Vec = v["properties"] + .as_object() + .unwrap() + .keys() + .filter(|k| k.as_str() != DBT_COMMAND_LABEL) + .cloned() + .collect(); + names.sort(); + (v.clone(), names) + }; + + let (build, build_args) = of("build"); + assert_eq!(build_args, ["exclude", "full_refresh", "select", "vars"]); + assert_eq!(build["properties"]["full_refresh"]["type"], "boolean"); + // Defaults come from the descriptor, so an untouched run reproduces it. + assert_eq!( + build["properties"]["select"]["default"], + serde_json::json!(["tag:nightly+"]) + ); + // The discriminator takes one value per variant: the toggle sets it, and + // a block cannot claim one command while carrying another's fields. + assert_eq!( + build["properties"][DBT_COMMAND_LABEL]["enum"], + serde_json::json!(["build"]) + ); + + let (retry, retry_args) = of("retry"); + assert_eq!(retry_args, ["dbt_retry_job"]); + // Required where it means something, rather than required everywhere and + // hidden: a retry names the run it resumes. + assert_eq!(retry["required"], serde_json::json!(["dbt_retry_job"])); + + // `show` is NOT a form variant: previewing one model's rows is something + // you do to a table you are looking at, and the graph and the assets list + // are where that lives. The worker still accepts it programmatically. + assert!( + !variants.iter().any(|v| v["title"] == "show"), + "show must not be offered in the run form: {schema}" + ); + assert!(DBT_COMMANDS.contains(&"show"), "but the worker still takes it"); + + // Every `{{ placeholder }}` the descriptor interpolates is an argument a + // run must supply. The command block is not one: it defaults to the + // descriptor's own selection under the descriptor's command. + assert_eq!(schema["required"], serde_json::json!(["day"])); + assert_eq!(schema["order"], serde_json::json!([DBT_COMMAND_ARG, "day"])); + } + + #[test] + fn parses_descriptor() { + let d = parse_dbt_descriptor(DESCRIPTOR).unwrap(); + assert_eq!(d.engine(), DbtEngine::DbtCore2x); + assert_eq!(d.profile.target.as_deref(), Some("prod")); + assert_eq!(d.select, vec!["tag:nightly+"]); + assert_eq!(d.threads, Some(8)); + assert!(d.full_refresh); + // dbt vars keep their YAML type: a `false` coerced to "false" is truthy + // in Jinja and would silently invert the condition it gates. + assert_eq!(d.vars["strict"], serde_json::json!(false)); + assert_eq!(d.vars["run_date"], serde_json::json!("{{ day }}")); + } + + #[test] + fn an_empty_descriptor_defaults_to_the_bundled_engine() { + let d = parse_dbt_descriptor("").unwrap(); + assert_eq!(d.engine(), DbtEngine::DbtCore1x); + } + + // The ORDER is asserted, not just the set: the schema's `order` is built from + // this vec and the run form follows it, so the command block leading is what + // puts the choice of what the run does above the project's own inputs. + #[test] + fn signature_exposes_the_command_block_and_placeholders() { + let sig = parse_dbt_sig(DESCRIPTOR).unwrap(); + let names: Vec<&str> = sig.args.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, vec![DBT_COMMAND_ARG, "day"]); + // An untouched run reproduces the descriptor: its default is the whole + // block — the descriptor's command, carrying the descriptor's selection. + let cmd = sig.args.iter().find(|a| a.name == DBT_COMMAND_ARG).unwrap(); + let default = cmd.default.clone().unwrap(); + // `build`, not `run`: `run` covers models only, so a selection naming a + // seed or a snapshot would silently not build it. + assert_eq!(default[DBT_COMMAND_LABEL], serde_json::json!("build")); + assert_eq!(default["select"], serde_json::json!(["tag:nightly+"])); + assert_eq!(default["full_refresh"], serde_json::json!(true)); + // `vars` is the exception: it overrides, so its default must be empty. + // Seeded with the descriptor, the run form would post `{{ day }}` back + // and overwrite the value the worker interpolated for it. + assert_eq!(default["vars"], serde_json::json!({})); + // Placeholders are required (the descriptor names no value for them) + // and untyped, so a `{{ }}` var can carry a boolean or a number rather + // than the string "false", which Jinja treats as truthy. + let day = sig.args.iter().find(|a| a.name == "day").unwrap(); + assert!(!day.has_default); + assert_eq!(day.typ, Typ::Unknown); + } +} + diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index f2a1a2b824..e025849769 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -9,6 +9,13 @@ use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; pub mod asset_parser; pub use asset_parser::parse_assets; +pub mod dbt; +pub use dbt::{ + dbt_arg_schema, default_command as default_dbt_command, parse_dbt_descriptor, parse_dbt_sig, + DbtDescriptor, DbtEngine, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, DBT_COMMAND_LABEL, + DBT_DEFAULT_WAREHOUSE, +}; + pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result { let docs = YamlLoader::load_from_str(inner_content) .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?; diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index de35ccdd6e..843a83a310 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -1,4 +1,5 @@ use serde::Serialize; +use std::borrow::Cow; use std::collections::BTreeMap; // Token recognized inside declared asset URIs that the runtime substitutes @@ -28,6 +29,11 @@ pub enum AssetKind { Ducklake, DataTable, Volume, + /// A warehouse relation a dbt project builds or reads, + /// `dbt:////`, the warehouse named as the + /// workspace configures it. The scheme names the producer, the path stays + /// the relation — see `windmill_types::AssetKind::Dbt`. + Dbt, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -714,21 +720,32 @@ pub fn asset_was_used(assets: &Vec, (kind, path): (AssetKind, }) } -pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(AssetKind, &str)> { +/// Split an asset URI into `(kind, path)`. The single point where user-written +/// asset URIs become graph keys, and therefore where `dbt://` paths get +/// canonicalized (see `canonicalize_table_asset_path`) — every other kind's +/// suffix is kept verbatim and stays borrowed. +pub fn parse_asset_syntax( + s: &str, + enable_default_syntax: bool, +) -> Option<(AssetKind, Cow<'_, str>)> { if enable_default_syntax && s == "datatable" { - return Some((AssetKind::DataTable, "main")); + return Some((AssetKind::DataTable, Cow::Borrowed("main"))); } else if enable_default_syntax && s == "ducklake" { - return Some((AssetKind::Ducklake, "main")); + return Some((AssetKind::Ducklake, Cow::Borrowed("main"))); } for (prefix, kind) in ASSET_KINDS.iter() { if s.starts_with(prefix) { + let suffix = &s[prefix.len()..]; + if *kind == AssetKind::Dbt { + return Some((*kind, Cow::Owned(canonicalize_table_asset_path(suffix)))); + } // The suffix is kept verbatim. For S3 the path encodes the storage: // `s3:///`, with an EMPTY storage segment for the // workspace default — so `s3:///key` yields `/key` (leading slash // significant, default storage) while `s3://secondary/key` yields // `secondary/key`. Stripping leading slashes here would conflate a // default-storage object with a named-storage one. - return Some((*kind, &s[prefix.len()..])); + return Some((*kind, Cow::Borrowed(suffix))); } } None @@ -741,8 +758,123 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("ducklake://", AssetKind::Ducklake), ("datatable://", AssetKind::DataTable), ("volume://", AssetKind::Volume), + ("dbt://", AssetKind::Dbt), ]; +/// Canonical spelling of a `dbt://` path, `//`. +/// +/// The whole point of keying warehouse relations on the relation rather than on +/// the producing tool is that a dbt mart and a native script reading the same +/// table land on one graph node. That only holds if both sides spell the key +/// identically, and they will not by default: dbt's `manifest.json` gives +/// `relation_name` pre-quoted (`"db"."Schema"."Tbl"`) while an annotation is +/// written by hand, and the warehouses disagree on case (Snowflake folds +/// unquoted identifiers up, Postgres folds them down, DuckDB compares them +/// case-insensitively). Two spellings of one table produce two nodes, no edge, +/// and nothing looks broken in isolation — so the rule is applied once, here, +/// on every path that becomes a `dbt://` asset. +/// +/// The rule: strip the quote characters the warehouses use (`"`, backtick, +/// `[`/`]`) from the schema and name, then ASCII-lowercase them. This matches +/// the case-insensitive identifier comparison the DuckDB paths already use +/// (`schema_contracts::CapturedSchema::find`). The warehouse-name prefix is +/// spelled as the workspace configures it, which is case-sensitive, and is left +/// untouched. +/// +/// Consequence to accept: a relation deliberately created under a quoted +/// mixed-case identifier collides with its lowercase spelling. That is rarer +/// than the case-fold mismatch it prevents, and it errs toward unifying nodes +/// rather than splitting them. +pub fn canonicalize_table_asset_path(path: &str) -> String { + let Some((resource, rest)) = path.rsplit_once('/').and_then(|(head, name)| { + head.rsplit_once('/') + .map(|(resource, schema)| (resource, (schema, name))) + }) else { + // Fewer than three segments: not a well-formed relation path. Leave it + // alone so the malformed value stays visible instead of being reshaped + // into something that looks valid. + return path.to_string(); + }; + let (schema, name) = rest; + format!( + "{}/{}/{}", + resource, + unquote_schema_segment(schema).to_ascii_lowercase(), + unquote_identifier(name).to_ascii_lowercase() + ) +} + +/// A doubled delimiter inside a quoted identifier is that delimiter, literally — +/// the same rule the worker's `split_relation` applies to `relation_name`. Both +/// have to decode it or one spelling of a table becomes two graph nodes: the dbt +/// model under `sales"east`, its native consumer under `sales""east`. +fn unquote_identifier(s: &str) -> String { + let s = s.trim(); + for (open, close) in [('"', '"'), ('`', '`'), ('[', ']')] { + if s.len() >= 2 && s.starts_with(open) && s.ends_with(close) { + let inner = &s[1..s.len() - 1]; + return inner.replace(&format!("{close}{close}"), &close.to_string()); + } + } + s.to_string() +} + +/// Unquote a schema segment, which carries a `.` pair when a +/// relation overrode its database. Each half is separately quotable, so +/// stripping only the outer pair leaves `"Archive"."Sales"` as +/// `Archive"."Sales` — a different key from the ingest's `archive.sales`, which +/// silently splits the model and its native consumer into two nodes. +/// +/// The halves go through `unquote_identifier`, which requires the quote at both +/// ends, because this function sees both a warehouse SPELLING (from a `dbt://` +/// annotation) and an already-DECODED identifier (from `table_asset_path`, whose +/// callers ran `split_relation` or read the manifest) and cannot tell them apart. +/// A lone `"` in a decoded name — `sa"les` — must survive: treated as opening a +/// quote it would be dropped, filing the ingest's `sa"les` under `sales` while +/// the annotation's `"sa""les"` decodes to `sa"les`, which is the split this +/// canonicalization exists to prevent. +fn unquote_schema_segment(s: &str) -> String { + let s = s.trim(); + let mut parts: Vec<&str> = Vec::new(); + let mut start = 0usize; + let mut quote: Option = None; + let bytes: Vec<(usize, char)> = s.char_indices().collect(); + let mut k = 0usize; + while k < bytes.len() { + let (i, c) = bytes[k]; + match quote { + Some(q) => { + let close = if q == '[' { ']' } else { q }; + if c == close { + // Doubled: a literal delimiter, so the quote stays open. + if bytes.get(k + 1).map(|(_, n)| *n) == Some(close) { + k += 1; + } else { + quote = None; + } + } + } + // Only a quote that OPENS the part can be a delimiter; one in the + // middle of a decoded identifier is part of the name. + None if (c == '"' || c == '`' || c == '[') && i == start => quote = Some(c), + // Only an UNQUOTED period separates the database from the schema; + // one inside an identifier is part of the name. + None if c == '.' => { + parts.push(&s[start..i]); + start = i + c.len_utf8(); + } + None => {} + } + k += 1; + } + parts.push(&s[start..]); + parts + .into_iter() + .map(unquote_identifier) + .collect::>() + .join(".") +} + // Tokenize a `key=value [key="quoted value"] ...` option string. Bare // values run until the next whitespace; quoted values consume until the // matching quote. Malformed pairs (missing `=` or empty key) are skipped @@ -1555,11 +1687,11 @@ mod pipeline_annotation_tests { // objects and must never collapse to one identity. assert_eq!( parse_asset_syntax("s3:///exports/x", false), - Some((AssetKind::S3Object, "/exports/x")) + Some((AssetKind::S3Object, Cow::Borrowed("/exports/x"))) ); assert_eq!( parse_asset_syntax("s3://exports/x", false), - Some((AssetKind::S3Object, "exports/x")) + Some((AssetKind::S3Object, Cow::Borrowed("exports/x"))) ); assert_ne!( parse_asset_syntax("s3:///exports/x", false), @@ -1569,28 +1701,181 @@ mod pipeline_annotation_tests { // The `// on` trigger annotation goes through the same function. assert_eq!( parse_asset_syntax("s3:///exports/x", true), - Some((AssetKind::S3Object, "/exports/x")) + Some((AssetKind::S3Object, Cow::Borrowed("/exports/x"))) ); assert_eq!( parse_asset_syntax("s3://secondary_storage/path/to/file.csv", false), - Some((AssetKind::S3Object, "secondary_storage/path/to/file.csv")) + Some(( + AssetKind::S3Object, + Cow::Borrowed("secondary_storage/path/to/file.csv") + )) ); // Hive-partition keys are preserved verbatim. assert_eq!( parse_asset_syntax("s3:///t/year=2024/month=01/f.parquet", false), - Some((AssetKind::S3Object, "/t/year=2024/month=01/f.parquet")) + Some(( + AssetKind::S3Object, + Cow::Borrowed("/t/year=2024/month=01/f.parquet") + )) ); // Non-S3 kinds also keep their suffix verbatim. assert_eq!( parse_asset_syntax("res://f/foo", false), - Some((AssetKind::Resource, "f/foo")) + Some((AssetKind::Resource, Cow::Borrowed("f/foo"))) ); assert_eq!( parse_asset_syntax("ducklake://analytics/orders", false), - Some((AssetKind::Ducklake, "analytics/orders")) + Some((AssetKind::Ducklake, Cow::Borrowed("analytics/orders"))) + ); + } + + // A dbt mart and a native script reading the same warehouse table must land + // on ONE graph node. They only do if every spelling of the relation + // canonicalizes identically — dbt's manifest gives it pre-quoted, the + // annotation is hand-written, and the warehouses fold case in opposite + // directions. A regression here is invisible: both nodes still render, they + // just stop being the same node and the cross-boundary cascade never fires. + #[test] + fn table_paths_from_every_spelling_canonicalize_to_one_key() { + let canonical = Some(( + AssetKind::Dbt, + Cow::Owned("main/analytics/orders".into()), + )); + for spelling in [ + // Hand-written annotation. + "dbt://main/analytics/orders", + // dbt manifest `relation_name`, quoted (database segment already + // dropped by the ingest, which keys on the warehouse name instead). + "dbt://main/\"analytics\"/\"orders\"", + // Snowflake folds unquoted identifiers up, Postgres folds down. + "dbt://main/ANALYTICS/ORDERS", + "dbt://main/Analytics/Orders", + // BigQuery / Databricks backticks, SQL Server brackets. + "dbt://main/`analytics`/`orders`", + "dbt://main/[Analytics]/[Orders]", + ] { + assert_eq!( + parse_asset_syntax(spelling, false), + canonical, + "spelling {spelling} did not canonicalize" + ); + } + } + + // A relation that overrode its database carries `.` in + // one segment, and each half can be quoted independently. Stripping only + // the outer pair leaves a key the manifest ingest never produces, so the + // model and a native script reading it become separate nodes. + #[test] + fn qualified_schema_segments_unquote_each_half() { + let canonical = Some(( + AssetKind::Dbt, + Cow::Owned("main/archive.sales/orders".into()), + )); + for spelling in [ + "dbt://main/archive.sales/orders", + "dbt://main/\"Archive\".\"Sales\"/\"Orders\"", + "dbt://main/`Archive`.`Sales`/`Orders`", + "dbt://main/[Archive].[Sales]/[Orders]", + ] { + assert_eq!( + parse_asset_syntax(spelling, false), + canonical, + "spelling {spelling} did not canonicalize" + ); + } + // A period INSIDE one quoted identifier is part of the name, not a + // database qualifier. + assert_eq!( + parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), + Some(( + AssetKind::Dbt, + Cow::Owned("main/sales.v2/orders".into()) + )) + ); + } + + // Every dialect here escapes its own delimiter by doubling it, and the + // worker's `split_relation` decodes it — so a canonicalizer that did not + // would file the dbt model under `sales"east` and the native script reading + // it under `sales""east`, which is the split this key exists to prevent. + #[test] + fn a_doubled_delimiter_canonicalizes_like_the_relation_it_names() { + for (spelling, expected) in [ + ( + "dbt://main/\"sales\"\"east\"/\"orders\"", + "main/sales\"east/orders", + ), + ( + "dbt://main/analytics/\"order\"\"s\"", + "main/analytics/order\"s", + ), + ( + "dbt://main/`da``ta`/`orders`", + "main/da`ta/orders", + ), + ( + "dbt://main/[my]]schema]/[orders]", + "main/my]schema/orders", + ), + // And in one half of a database-qualified segment. + ( + "dbt://main/\"arch\"\"ive\".\"sales\"/orders", + "main/arch\"ive.sales/orders", + ), + ] { + assert_eq!( + parse_asset_syntax(spelling, false), + Some((AssetKind::Dbt, Cow::Owned(expected.into()))), + "spelling {spelling} did not canonicalize" + ); + } + // The DECODED form has to land on the same key, in both halves: this + // function sees the warehouse's spelling from an annotation and an + // already-decoded identifier from the ingest, and cannot tell them + // apart. A lone delimiter treated as opening a quote would be dropped — + // `sa"les` filed as `sales` — and the two derivations would split. + for (decoded, spelled) in [ + ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), + ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/arch\"ive.sales/orders", + "dbt://main/\"arch\"\"ive\".\"sales\"/orders", + ), + ] { + assert_eq!( + parse_asset_syntax(decoded, false), + parse_asset_syntax(spelled, false), + "the decoded and quoted spellings of {decoded} disagree" + ); + } + } + + #[test] + fn table_canonicalization_leaves_the_warehouse_name_case_alone() { + // The warehouse-name prefix is spelled as the workspace configures it + // and is case-sensitive; only the two identifier segments are folded. + assert_eq!( + parse_asset_syntax("dbt://MyWarehouse/Sales/Orders", false), + Some(( + AssetKind::Dbt, + Cow::Owned("MyWarehouse/sales/orders".into()) + )) + ); + // A database-qualified schema keeps its own case rules: the whole + // segment folds, dot included. + assert_eq!( + parse_asset_syntax("dbt://main/PROD.Sales/Orders", false), + Some((AssetKind::Dbt, Cow::Owned("main/prod.sales/orders".into()))) + ); + // Too few segments to be a relation: left alone rather than reshaped + // into something that looks well-formed. + assert_eq!( + parse_asset_syntax("dbt://Orders", false), + Some((AssetKind::Dbt, Cow::Owned("Orders".into()))) ); } diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 1936f7ff70..9493a78606 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -110,6 +110,7 @@ fn kind_str(k: AssetKind) -> &'static str { AssetKind::Ducklake => "ducklake", AssetKind::DataTable => "datatable", AssetKind::Volume => "volume", + AssetKind::Dbt => "dbt", } } diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index b47a6556cd..96ce7811a2 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -3,7 +3,7 @@ ## ENUMs action_kind: create, update, delete, execute asset_access_type: r, w, rw -asset_kind: s3object, resource, variable, ducklake, datatable +asset_kind: s3object, resource, variable, ducklake, datatable, table asset_usage_kind: script, flow, job authentication_method: none, windmill, api_key, basic_http, custom_script, signature autoscaling_event_type: full_scaleout, scalein, scaleout @@ -19,6 +19,7 @@ job_status: success, failure, canceled, skipped job_trigger_kind: webhook, http, websocket, kafka, email, nats, schedule, app, ui, postgres, sqs, gcp, mqtt, nextcloud, ci_test log_mode: standalone, server, worker, agent, indexer, mcp login_type: password, github +materialization_status: running, materialized, failed message_type: user, assistant, tool metric_kind: scalar_int, scalar_float, timeseries_int, timeseries_float mqtt_client_version: v3, v5 @@ -26,7 +27,7 @@ native_trigger_service: nextcloud request_type: sync, async, sync_sse runnable_type: ScriptHash, ScriptPath, FlowPath script_kind: script, trigger, failure, command, approval, preprocessor -script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby, rlang +script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby, rlang, dbt trigger_kind: webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp, default_email, nextcloud trigger_mode: enabled, disabled, suspended workspace_key_kind: cloud @@ -65,6 +66,16 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) +dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) + FK: (workspace_id) -> workspace(id) +dbt_run_state: workspace_id(char), script_path(char), permissioned_as(char), identity(text), args(jsonb), run_results(text), job_id(uuid), retryable(bool), updated_at(ts) + FK: (workspace_id) -> workspace(id) debounce_key: key(char), job_id(uuid), previous_job_id(uuid), first_started_at(ts), debounced_times(int) debounce_stale_data: job_id(uuid), to_relock(text[]) debouncing_settings: hash(bigint), debounce_key(char), debounce_delay_s(int), max_total_debouncing_time(int), max_total_debounces_amount(int), debounce_args_to_accumulate(text[]) @@ -204,7 +215,7 @@ workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_gr FK: (workspace_id) -> workspace(id) workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/windmill-api-assets/Cargo.toml b/backend/windmill-api-assets/Cargo.toml index c75715a3ac..62bc3d9c8d 100644 --- a/backend/windmill-api-assets/Cargo.toml +++ b/backend/windmill-api-assets/Cargo.toml @@ -21,3 +21,7 @@ serde.workspace = true serde_json.workspace = true sqlx.workspace = true tracing.workspace = true +uuid.workspace = true + +[dev-dependencies] +tokio = { workspace = true } diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 3a2bf83097..940b610113 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -605,9 +605,16 @@ async fn list_favorites( // frontend aggregates into nodes and edges. #[derive(Deserialize)] -struct GraphQuery { +pub struct GraphQuery { pub asset_kinds: Option, pub folder: Option, + /// Render a dbt project as a given deployed version rather than as it is + /// now. A run page passes the version its job ran, so an old run shows the + /// models, SQL and `ref()` lineage of that deploy instead of today's. + /// Absent — the usual case — means the newest live version per path. + /// Hex, like every other script-hash parameter — `ScriptHash` deserializes + /// it, so the run page can pass `job.script_hash` verbatim. + pub dbt_script_hash: Option, } #[derive(Serialize, Debug)] @@ -629,6 +636,68 @@ struct GraphAssetNode { // `AssetGraphAssetNode.derived_from`. #[serde(skip_serializing_if = "Option::is_none")] derived_from: Option, + // Set on a `dbt://` asset produced (or, for a source, consumed) by a dbt + // script: which dbt node it is and what dbt says about it. A dbt project is + // one runnable node with many model assets, so per-model metadata belongs + // here rather than on the script (docs/dbt-runtime.md, decision 15). + // Lockstep with TS `AssetGraphAssetNode.dbt`. + #[serde(skip_serializing_if = "Option::is_none")] + dbt: Option, +} + +#[derive(Serialize, Debug, Clone)] +struct DbtAssetProvenance { + unique_id: String, + // `model` | `snapshot` | `seed` | `source` — a source is read, not written, + // and the canvas distinguishes them. + resource_type: String, + // dbt's own word (`table`, `view`, `incremental`, `snapshot`), kept because + // `view` and `ephemeral` have no Windmill write-strategy analogue. + #[serde(skip_serializing_if = "Option::is_none")] + materialized: Option, + #[serde(skip_serializing_if = "Option::is_none")] + materialize_strategy: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + tags: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + data_tests: Vec, + /// Declared column metadata (name -> description). NOT column lineage — + /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14). + #[serde(skip_serializing_if = "Option::is_none")] + columns: Option, + /// A source's declared freshness policy, for the staleness chip. + #[serde(skip_serializing_if = "Option::is_none")] + freshness: Option, + /// The model's SQL as written. Read-only in Windmill — the file lives in + /// the repo at the pinned commit, and this is a copy taken at deploy. + #[serde(skip_serializing_if = "Option::is_none")] + raw_code: Option, + /// Its path inside the repo, e.g. `models/staging/stg_orders.sql`. + #[serde(skip_serializing_if = "Option::is_none")] + original_file_path: Option, +} + +#[derive(Serialize, Debug, Clone, PartialEq)] +struct DbtDataTest { + // `unique` | `not_null` | `accepted_values` | `relationships` — the four + // generic tests, one-for-one with the `// data_test` kinds — or a package + // test's namespaced name (`dbt_utils.accepted_range`). + kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + column: Option, + #[serde(skip_serializing_if = "Option::is_none")] + args: Option, + // Lowercased. dbt's own severity decides whether a failure fails the run, + // so the canvas shows it rather than assuming every test is blocking. + #[serde(skip_serializing_if = "Option::is_none")] + severity: Option, +} + +#[derive(Serialize, Debug, Clone)] +struct DbtRunnableProvenance { + model_count: usize, } #[derive(Serialize, Debug)] @@ -691,6 +760,11 @@ struct GraphRunnableNode { // signature list. Lockstep with TS `AssetGraphRunnableNode.macros`. #[serde(skip_serializing_if = "Vec::is_empty", default)] macros: Vec, + // Set on a `ScriptLang::Dbt` script: it owns a whole dbt project, so the + // node says how many models it materializes rather than pretending to be a + // single-output script. Lockstep with TS `AssetGraphRunnableNode.dbt`. + #[serde(skip_serializing_if = "Option::is_none", default)] + dbt: Option, } // One macro of a `// macros` library, as surfaced on its graph node. @@ -831,7 +905,7 @@ struct TestEdge { } #[derive(Serialize, Debug)] -struct AssetGraphResponse { +pub struct AssetGraphResponse { assets: Vec, runnables: Vec, edges: Vec, @@ -840,6 +914,27 @@ struct AssetGraphResponse { macro_edges: Vec, #[serde(skip_serializing_if = "Vec::is_empty", default)] test_edges: Vec, + /// `ref()` lineage BETWEEN two dbt models. Without it every model hangs off + /// the one dbt runnable, which reads as a flat source-to-model fan-out + /// instead of the project's actual shape. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + dbt_edges: Vec, + /// The job whose snapshot the dbt half was resolved from, when one was + /// asked for and found. A run page polls the graph while its job runs + /// because a dynamic descriptor's snapshot is written mid-run, and this is + /// what tells it to stop: without it the page cannot distinguish "the + /// snapshot has not been written yet" from "this run has none", and either + /// polls forever or gives up before the ingest. + #[serde(skip_serializing_if = "Option::is_none")] + dbt_snapshot_job: Option, +} + +/// One `ref()`/`source()` edge, in the terms the canvas draws: the two +/// relations, not dbt's node ids. +#[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct DbtLineageEdge { + from_asset_path: String, + to_asset_path: String, } async fn asset_graph( @@ -849,15 +944,75 @@ async fn asset_graph( Extension(db): Extension, Query(q): Query, ) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; + // `None`: pinning the graph to one run is job-scoped and this endpoint is + // authorized as `assets:read`. See `asset_graph_for`. + asset_graph_for(&authed, &w_id, user_db, db, q, None).await +} - let kind_filter: Option> = q.asset_kinds.as_ref().map(|s| { - s.split(',') - .filter_map(|k| { - serde_json::from_value::(Value::String(k.trim().into())).ok() - }) - .collect() - }); +/// A run the caller is already authorized to read, and the deployed version it +/// ran. Path and hash come from the job row rather than the query, so a graph +/// cannot be pointed at one project's version while claiming another's run. +pub struct PinnedRun { + pub job_id: uuid::Uuid, + pub script_path: String, + pub script_hash: i64, +} + +/// The asset graph, optionally as one run saw it. +/// +/// AUTHORIZES NOTHING BY ITSELF. Every caller owes it two checks, because the +/// answer is workspace asset data reached through a route whose own URL segment +/// decides the scope domain: `assets:read` always, and `require_job_read_access` +/// for the job when passing `Some(pinned)` — which is then taken as already +/// done, since the pinned path and hash come from that job's row. +pub async fn asset_graph_for( + authed: &ApiAuthed, + w_id: &str, + user_db: UserDB, + db: windmill_common::DB, + q: GraphQuery, + pinned: Option, +) -> JsonResult { + let dbt_job_id = pinned.as_ref().map(|p| p.job_id); + // The version is the job's own, not the caller's `dbt_script_hash`. + let dbt_script_hash = pinned + .as_ref() + .map(|p| p.script_hash) + .or(q.dbt_script_hash.map(|h| h.0)); + // Set only for a pinned run, where it lets `live` resolve without reading + // `script`: a share-link viewer is entitled to the run but usually has no + // grant on the script, and RLS there would empty the graph. `raw_code` has + // its own `script` check, so the body stays hidden either way. + let pinned_path = pinned.as_ref().map(|p| p.script_path.as_str()); + let w_id = w_id.to_string(); + let mut tx = user_db.begin(authed).await?; + // Built once: a scoped token's `scripts:read` paths decide whether a dbt + // node's SQL body may be returned, independently of the `assets:read` scope + // that authorizes this endpoint. + let dbt_source_scope = build_scope_path_predicate(authed, "scripts", "read"); + + // REFUSED, not dropped: a caller that asks for a kind this server does not + // know has asked for something, and answering with the other kinds returns a + // graph missing exactly what they came for — silently. That is how a rename + // of one kind emptied three callers' graphs with nothing logged. + let kind_filter: Option> = q + .asset_kinds + .as_ref() + .map(|s| { + s.split(',') + .map(|k| { + serde_json::from_value::(Value::String(k.trim().into())).map_err( + |_| { + windmill_common::error::Error::BadRequest(format!( + "`{}` is not an asset kind", + k.trim() + )) + }, + ) + }) + .collect::, _>>() + }) + .transpose()?; let kind_filter_ref = kind_filter.as_deref(); let folder_filter = q.folder.as_deref().map(|f| format!("f/{}/%", f)); @@ -1066,6 +1221,218 @@ async fn asset_graph( .fetch_all(&mut *tx) .await?; + // dbt provenance. A dbt script is one runnable node whose models are many + // `dbt://` asset nodes (decision 15), so the per-model metadata has to + // hang off the assets, not off the script. It describes the RELATION, not a + // producer: several dbt scripts (different selections of one project) can + // materialize the same model, and the producer edges already name them. + // + // Scoped to the relations this graph actually renders, so a workspace with + // many dbt projects does not pay for all of them on every request. Not + // scoped by the script's folder: an out-of-folder dbt script still has to + // explain a model an in-scope consumer reads, same as macro definitions. + // Tests come along via `attached_node` — they carry no `asset_path` of + // their own. + let dbt_rows = sqlx::query!( + r#"WITH live AS ( + -- The graph is stored per deployed VERSION, so this endpoint — which + -- describes the project as it is now — takes the newest live one per + -- path. Resolved once here rather than per row: a correlated lookup + -- on every node is what makes these queries fall over. + SELECT * FROM ( + SELECT DISTINCT ON (s.path) s.path, s.hash + FROM script s + WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt' + AND ($3::bigint IS NULL OR s.hash = $3) + -- A pinned version may be archived by now; that is precisely + -- the case a historical run needs, so the liveness filter + -- applies only when picking the current one. + AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false)) + ORDER BY s.path, s.created_at DESC + ) cur + UNION ALL + -- A pinned run names its own version, so `script` is not consulted: + -- under RLS it would answer for the CALLER's grants on the project, + -- emptying the graph for a share-link viewer who is entitled to the + -- run but not the script. + SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL + ), + -- The run's own snapshot when it left one, the version's graph + -- otherwise. A static descriptor never snapshots, so all of its runs + -- fall through to the same rows. Existence comes from the marker, not + -- from a node row: a dynamic run that disabled every model has a + -- snapshot whose graph is legitimately empty. + chosen AS ( + -- No visibility check on the job here: reaching this with a job at + -- all means the caller passed `require_job_read_access` for it, and + -- re-deciding it under plain RLS can only DISAGREE with that answer + -- — silently, by falling back to the deployed graph rather than + -- erroring. A share-link viewer is entitled to the run and would be + -- shown a different run's model set. See `asset_graph_for`. + SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $4) + THEN $4::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id + ), + scoped AS ( + SELECT n.script_path, n.unique_id FROM dbt_node n + JOIN live l ON l.path = n.script_path AND l.hash = n.script_hash + JOIN chosen ch ON ch.job_id = n.job_id + WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL + -- Unpinned, the scope is the relations in view: `asset` says + -- which of them this folder touches. Pinned, that table is the + -- WRONG scope — it holds one row set per path, describing the + -- current deploy, so a model this version had and the current one + -- dropped would be filtered out of its own run's graph. The + -- pinned version's nodes are the scope. + AND ($3::bigint IS NOT NULL OR n.asset_path IN ( + SELECT path FROM asset + WHERE workspace_id = $1 AND kind = 'dbt' + AND ($2::text IS NULL OR usage_path LIKE $2))) + ) + SELECT n.script_path AS "script_path!", n.unique_id AS "unique_id!", + n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path, + n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description, + n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node, + n.columns, n.freshness, + n.raw_code, n.original_file_path, + -- Whether the caller may read the project this row describes. + -- The query deliberately reaches outside the requested folder + -- so an in-scope consumer can explain the relation it reads, + -- and `dbt_node` carries no RLS of its own; the relation's + -- SHAPE is fine to answer that way, everything the project's + -- author WROTE is not. Applied in Rust, over one predicate, so + -- the fields it covers are named in one place. This runs in the + -- authed transaction, so `script`'s RLS answers it. Matched on + -- the HASH as well: `extra_perms` is per row, so a path + -- recreated with narrower ones leaves the archived version + -- readable, and a path-only probe would answer for THAT grant + -- while returning this version's source. + EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path + AND sc.hash = n.script_hash + ) AS "script_visible!" + FROM dbt_node n + JOIN live l ON l.path = n.script_path AND l.hash = n.script_hash + -- Every join onto `dbt_node` needs this, not just the scoping CTE: + -- `job_id` is part of the key, so without it each model comes back + -- once per retained snapshot plus once for the version's graph. + JOIN chosen ch ON ch.job_id = n.job_id + WHERE n.workspace_id = $1 + -- Joined on BOTH columns: a dbt `unique_id` is project-local, so + -- two projects with the same model name would otherwise pull each + -- other's rows. + AND (EXISTS (SELECT 1 FROM scoped s + WHERE s.script_path = n.script_path + AND s.unique_id = n.unique_id) + OR EXISTS (SELECT 1 FROM scoped s + WHERE s.script_path = n.script_path + AND s.unique_id = n.attached_node)) + ORDER BY n.script_path, n.unique_id"#, + &w_id, + folder_filter.as_deref(), + dbt_script_hash, + dbt_job_id, + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + // `ref()` lineage between two models, resolved to the relations they + // produce. Joined to `dbt_node` on both key columns because a dbt + // `unique_id` is only unique within its project. + let dbt_edge_rows = sqlx::query!( + r#"WITH live AS ( + SELECT * FROM ( + SELECT DISTINCT ON (s.path) s.path, s.hash + FROM script s + WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt' + AND ($3::bigint IS NULL OR s.hash = $3) + AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false)) + ORDER BY s.path, s.created_at DESC + ) cur + UNION ALL + SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL + ), + -- The run's own snapshot when it left one, the version's graph + -- otherwise. A static descriptor never snapshots, so all of its runs + -- fall through to the same rows. Existence comes from the marker, not + -- from a node row: a dynamic run that disabled every model has a + -- snapshot whose graph is legitimately empty. + chosen AS ( + -- No visibility check on the job here: reaching this with a job at + -- all means the caller passed `require_job_read_access` for it, and + -- re-deciding it under plain RLS can only DISAGREE with that answer + -- — silently, by falling back to the deployed graph rather than + -- erroring. A share-link viewer is entitled to the run and would be + -- shown a different run's model set. See `asset_graph_for`. + SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $4) + THEN $4::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id + ) + SELECT p.asset_path AS "from_path!", c.asset_path AS "to_path!" + FROM dbt_edge e + JOIN live l ON l.path = e.script_path AND l.hash = e.script_hash + JOIN chosen ch ON ch.job_id = e.job_id + JOIN dbt_node p ON p.workspace_id = e.workspace_id + AND p.script_path = e.script_path + AND p.script_hash = e.script_hash + AND p.job_id = ch.job_id + AND p.unique_id = e.parent_unique_id + JOIN dbt_node c ON c.workspace_id = e.workspace_id + AND c.script_path = e.script_path + AND c.script_hash = e.script_hash + AND c.job_id = ch.job_id + AND c.unique_id = e.child_unique_id + WHERE e.workspace_id = $1 + AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL + -- Tests attach to their model as a badge, not as a lineage edge. + AND c.resource_type <> 'test' + -- Scoped by the RELATIONS, like the node provenance above, not by + -- the producing script's folder: two tables consumed in this + -- folder but produced by a dbt project outside it would otherwise + -- both render with their `ref()` edge missing. + -- Same as the node scope: pinned, `asset` describes the CURRENT + -- deploy, so gating on it drops the edges of models this version + -- had and a later one removed. The pinned version's own edges are + -- the answer. + AND ($3::bigint IS NOT NULL OR EXISTS ( + SELECT 1 FROM asset a + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.path = c.asset_path + AND ($2::text IS NULL OR a.usage_path LIKE $2)))"#, + &w_id, + folder_filter.as_deref(), + dbt_script_hash, + dbt_job_id, + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + // The same predicate `chosen` applies, answered once for the caller: it is + // what lets a run page stop polling. No `v2_job` recheck, for the reason + // `chosen` gives — disagreeing with the route's decision here would leave a + // share-link viewer with the right graph and a null marker, polling it 40 + // times over. + let dbt_snapshot_job = match dbt_job_id { + Some(job) => { + sqlx::query_scalar!( + "SELECT g.job_id FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $2 + LIMIT 1", + &w_id, + job, + ) + .fetch_optional(&mut *tx) + .await? + } + None => None, + }; tx.commit().await?; // Parse each pipeline member's body once into its badge annotations, keyed @@ -1110,6 +1477,138 @@ async fn asset_graph( (r.path.clone(), lineage) }) .collect(); + // Model → its provenance, and script → how many models it owns. Tests are + // folded onto the model they are attached to, which is what makes dbt's + // four generic tests render through the existing data-test node. + let mut dbt_by_asset_path: std::collections::HashMap = + Default::default(); + let mut dbt_model_count: std::collections::HashMap = Default::default(); + // Which relations each dbt script actually BUILDS. The sidecar also holds a + // row for a parent kept only to anchor a cross-selection edge, which the + // script reads rather than materializes — counting those would make a script + // selecting one mart claim the staging models upstream of it, and the badge + // says "materializes N models". + let dbt_writes: std::collections::HashSet<(&str, &str)> = rows + .iter() + .filter(|r| { + r.asset_kind == AssetKind::Dbt + && matches!(r.access_type.as_deref(), Some("w") | Some("rw")) + }) + .map(|r| (r.usage_path.as_str(), r.asset_path.as_str())) + .collect(); + // Keyed by (script_path, unique_id) — the sidecar's own primary key — since + // a dbt `unique_id` is only unique within its project. + let dbt_asset_path_by_unique_id: std::collections::HashMap<(&str, &str), &str> = dbt_rows + .iter() + .filter_map(|r| { + Some(( + (r.script_path.as_str(), r.unique_id.as_str()), + r.asset_path.as_deref()?, + )) + }) + .collect(); + for r in &dbt_rows { + let Some(asset_path) = r.asset_path.as_deref() else { + continue; + }; + if r.resource_type != "source" && dbt_writes.contains(&(r.script_path.as_str(), asset_path)) + { + *dbt_model_count.entry(r.script_path.clone()).or_default() += 1; + } + // What the project's author WROTE — the model's SQL, its path in the + // repo, the prose and labels around it — as opposed to the shape of the + // relation it produces. RLS says whether the caller may see the script, + // but not whether a scoped token may: this endpoint is authorized as + // `assets:read`, so a token deliberately narrowed to it would otherwise + // read a project outside its `scripts:read` paths. Both answers gate the + // same set of fields, because a share-link viewer entitled to the RUN is + // not thereby entitled to the documentation of a project they cannot + // open. + let source_allowed = r.script_visible && dbt_source_scope(&r.script_path); + let candidate = DbtAssetProvenance { + raw_code: r.raw_code.clone().filter(|_| source_allowed), + original_file_path: r.original_file_path.clone().filter(|_| source_allowed), + unique_id: r.unique_id.clone(), + resource_type: r.resource_type.clone(), + materialized: r.materialized.clone(), + materialize_strategy: r.materialize_strategy.clone(), + tags: if source_allowed { + r.tags.clone() + } else { + vec![] + }, + description: r.description.clone().filter(|_| source_allowed), + data_tests: vec![], + columns: r.columns.clone().filter(|_| source_allowed), + freshness: r.freshness.clone().filter(|_| source_allowed), + }; + // One relation can carry rows from several projects — typically a model + // in one and a source declaring it in another. The producer describes + // the relation and the source only names it, so which one wins must not + // depend on script-path ordering. The source's freshness policy is + // still worth keeping, so it fills in rather than overwrites. + let existing = dbt_by_asset_path + .entry(asset_path.to_string()) + .or_insert_with(|| candidate.clone()); + if existing.unique_id == candidate.unique_id { + continue; + } + let wins = match ( + existing.resource_type.as_str(), + candidate.resource_type.as_str(), + ) { + ("source", t) if t != "source" => true, + (t, "source") if t != "source" => false, + // Two rows of the same nature: pick by id so the graph does not + // change shape between requests. + _ => candidate.unique_id < existing.unique_id, + }; + let freshness = existing + .freshness + .clone() + .or_else(|| candidate.freshness.clone()); + if wins { + *existing = candidate; + } + existing.freshness = freshness; + } + // Tests fold onto the model they assert, which is what makes dbt's four + // generic tests render through the existing data-test node. A separate pass + // because rows arrive in `unique_id` order, so a test can precede its model. + for r in &dbt_rows { + if r.resource_type != "test" { + continue; + } + let Some(target) = r + .attached_node + .as_deref() + .and_then(|n| dbt_asset_path_by_unique_id.get(&(r.script_path.as_str(), n))) + else { + continue; + }; + let Some(entry) = dbt_by_asset_path.get_mut(*target) else { + continue; + }; + let test = DbtDataTest { + kind: r.test_kind.clone().unwrap_or_else(|| r.name.clone()), + column: r.test_column.clone(), + // The test's kind and column are the badge's shape, already spelled + // out by its `unique_id`. Its arguments are authored data — an + // `accepted_values` list is a column's domain — so they follow the + // same gate as the model's own source. + args: r + .test_args + .clone() + .filter(|_| r.script_visible && dbt_source_scope(&r.script_path)), + // dbt-core 1.x echoes the author's casing, 2.x uppercases; fold so + // the badge reads the same whichever engine deployed the script. + severity: r.severity.as_deref().map(|s| s.to_ascii_lowercase()), + }; + if !entry.data_tests.contains(&test) { + entry.data_tests.push(test); + } + } + let last_success_by_path: std::collections::HashMap> = last_success_rows .into_iter() @@ -1132,6 +1631,17 @@ async fn asset_graph( let mut edges = Vec::with_capacity(rows.len()); let mut asset_set: std::collections::HashSet<(AssetKind, String)> = Default::default(); + // Pinned to a version, the relations come from that version's own nodes. The + // `asset` rows above are path-keyed — one set per script, always the current + // deploy — so a model this version had and a later one dropped would be + // missing from its own run's graph. + if dbt_script_hash.is_some() { + for r in &dbt_rows { + if let Some(p) = r.asset_path.as_deref() { + asset_set.insert((AssetKind::Dbt, p.to_string())); + } + } + } let mut runnable_set: std::collections::HashSet<(AssetUsageKind, String)> = Default::default(); // Every pipeline member in scope goes into the graph, even when the parser @@ -1491,6 +2001,9 @@ async fn asset_graph( }) .flatten(), derived_from: scd2_current_base.get(&(kind, path.clone())).cloned(), + dbt: (kind == AssetKind::Dbt) + .then(|| dbt_by_asset_path.get(&path).cloned()) + .flatten(), kind, path, }) @@ -1562,6 +2075,10 @@ async fn asset_graph( .flatten() .cloned() .unwrap_or_default(), + dbt: (usage_kind == AssetUsageKind::Script) + .then(|| dbt_model_count.get(&path)) + .flatten() + .map(|n| DbtRunnableProvenance { model_count: *n }), path, usage_kind, } @@ -1569,6 +2086,22 @@ async fn asset_graph( .collect(); runnables.sort_by(|a, b| a.path.cmp(&b.path)); + // Only between relations this graph actually renders — an edge to a node + // that is not here has nothing to draw. + let rendered: std::collections::HashSet<&str> = + assets.iter().map(|a| a.path.as_str()).collect(); + let mut dbt_edges: Vec = dbt_edge_rows + .into_iter() + .filter(|r| { + r.from_path != r.to_path + && rendered.contains(r.from_path.as_str()) + && rendered.contains(r.to_path.as_str()) + }) + .map(|r| DbtLineageEdge { from_asset_path: r.from_path, to_asset_path: r.to_path }) + .collect(); + dbt_edges.sort(); + dbt_edges.dedup(); + Ok(Json(AssetGraphResponse { assets, runnables, @@ -1576,6 +2109,8 @@ async fn asset_graph( triggers, macro_edges, test_edges, + dbt_edges, + dbt_snapshot_job, })) } diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs new file mode 100644 index 0000000000..d5e1418b78 --- /dev/null +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -0,0 +1,348 @@ +//! What a caller who cannot see a dbt script gets from a run of it. +//! +//! The share-link case: `require_job_read_access` has already let them through +//! to the job, so `asset_graph_for` is handed the run — but the project itself +//! is not theirs to read. The graph's SHAPE must survive that and its model SQL +//! must not, and the two are decided by different predicates. Resolving the +//! version from the caller's `script` access instead of from the job emptied the +//! whole dbt half, and every later fix in this area re-touched one of the two. + +use sqlx::{Pool, Postgres}; +use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun}; +use windmill_api_auth::ApiAuthed; +use windmill_common::db::UserDB; + +const WS: &str = "test-workspace"; +const PATH: &str = "f/private/proj"; +const HASH: i64 = 42; + +/// A member of the workspace with no grant on `f/private` — the reason someone +/// is sent a share link in the first place. +fn outsider() -> ApiAuthed { + ApiAuthed { + email: "outsider@windmill.dev".to_string(), + username: "outsider".to_string(), + is_admin: false, + is_operator: false, + groups: vec![], + folders: vec![], + scopes: None, + username_override: None, + username_override_is_token_label: false, + is_session_token: false, + token_prefix: None, + read_only: false, + } +} + +async fn seed(db: &Pool, job: uuid::Uuid) { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) + VALUES ($1, 'private', 'private', '{}', '{}')", + WS + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + HASH, + PATH, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest) + VALUES ($1, $2, $3, $4, 'd')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, raw_code, tags, description, + columns, freshness) + VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders', + 'u/a/wh/analytics/orders', 'select 1', '{finance}', 'daily order facts', + '{"order_id": {"description": "natural key"}}'::jsonb, + '{"warn_after": {"count": 12, "period": "hour"}}'::jsonb)"#, + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + // A test node, for the arguments it carries: `accepted_values` spells out a + // column's domain. + sqlx::query!( + r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, tags, test_kind, test_column, test_args, + attached_node) + VALUES ($1, $2, $3, $4, 'test.p.accepted_values_orders_status', 'test', + 'accepted_values_orders_status', '{}', 'accepted_values', 'status', + '{"values": ["gold", "silver"]}'::jsonb, 'model.p.orders')"#, + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); +} + +/// Everything on a node that the project's author wrote, rather than the shape +/// of the relation it produces. +const AUTHORED: [&str; 5] = [ + "select 1", + "daily order facts", + "finance", + "natural key", + "gold", +]; + +fn query() -> GraphQuery { + GraphQuery { asset_kinds: Some("dbt".to_string()), folder: None, dbt_script_hash: None } +} + +/// Pinned to a run they are entitled to, the outsider gets the model — and +/// nothing its author wrote. Both halves matter: dropping the first is the blank +/// Models panel under working progress rows, dropping the second hands the +/// project's source and documentation to anyone holding a link. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_pinned_run_survives_no_access_to_its_script(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + + let pinned = PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: HASH }; + let res = asset_graph_for( + &outsider(), + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(pinned), + ) + .await + .unwrap(); + let body = serde_json::to_value(&res.0).unwrap(); + + let nodes = body["dbt_nodes"] + .as_array() + .or(body["assets"].as_array()) + .unwrap(); + assert!( + !nodes.is_empty(), + "a run the caller may read must render, whatever their access to the project: {body}" + ); + assert_eq!( + body["dbt_snapshot_job"], + serde_json::json!(job), + "and the marker must agree with it, or the page polls the graph 40 times" + ); + for authored in AUTHORED { + assert!( + !body.to_string().contains(authored), + "`{authored}` is the project's, and stays behind access to it: {body}" + ); + } + assert!( + body.to_string().contains("u/a/wh/analytics/orders"), + "while the relation the run wrote is what the page is for: {body}" + ); + + // The same read by someone who may open the project: the gate has to be the + // caller's access, not a field this endpoint stopped serving. + let seen = asset_graph_for( + &ApiAuthed { is_admin: true, ..outsider() }, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: HASH }), + ) + .await + .unwrap(); + let seen = serde_json::to_value(&seen.0).unwrap().to_string(); + for authored in AUTHORED { + assert!( + seen.contains(authored), + "`{authored}` renders for a reader of the project: {seen}" + ); + } +} + +/// Unpinned, the same caller sees nothing of the project: the workspace graph +/// answers for their own access, and this one is not theirs. This is the half +/// that must NOT be relaxed by making the pinned case work. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn without_a_run_the_same_caller_sees_no_dbt_nodes(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + + let res = asset_graph_for( + &outsider(), + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + None, + ) + .await + .unwrap(); + let body = serde_json::to_value(&res.0).unwrap(); + assert!( + !body.to_string().contains("orders"), + "an unpinned graph is the caller's own view of the workspace: {body}" + ); +} + +/// Archiving retires the script; it must not retire the runs that already +/// happened. The pinned read resolves versions through a CTE that skips archived +/// rows, so nothing about the CURRENT workspace graph depends on the sidecar +/// surviving — which is exactly why deleting it looks safe and is not. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn archiving_the_script_leaves_its_finished_runs_renderable(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + sqlx::query!( + "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + WS, + HASH + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: HASH }; + let res = asset_graph_for( + &admin, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(pinned), + ) + .await + .unwrap(); + assert!( + serde_json::to_value(&res.0) + .unwrap() + .to_string() + .contains("orders"), + "a completed run of an archived version still renders its models" + ); + + let now = asset_graph_for( + &admin, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + None, + ) + .await + .unwrap(); + assert!( + !serde_json::to_value(&now.0) + .unwrap() + .to_string() + .contains("model.p.orders"), + "while the workspace graph, which describes what is live, drops it" + ); +} + +/// `extra_perms` is a grant on a ROW, so a path recreated with narrower ones +/// leaves the old version readable to whoever the old row named. The source +/// probe therefore has to name the version it is about to return: matched on +/// the path alone, that stale grant answered for the pinned version's SQL and +/// handed a viewer the body of a project they were never given. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_old_grant_at_the_same_path_does_not_expose_a_newer_version(db: Pool) { + const V2: i64 = 43; + let v1_job = uuid::Uuid::from_u128(7); + let v2_job = uuid::Uuid::from_u128(8); + seed(&db, v1_job).await; + + // The version the outsider WAS granted, archived — the shape the grant + // outlives the deploy in. + sqlx::query!( + r#"UPDATE script SET archived = true, extra_perms = '{"u/outsider": true}'::jsonb + WHERE workspace_id = $1 AND hash = $2"#, + WS, + HASH + ) + .execute(&db) + .await + .unwrap(); + + // The version they were not. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + V2, + PATH, + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest) + VALUES ($1, $2, $3, $4, 'd2')", + WS, + PATH, + V2, + v2_job + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, raw_code, tags) + VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders', + 'u/a/wh/analytics/orders', 'select 2', '{}')", + WS, + PATH, + V2, + v2_job + ) + .execute(&db) + .await + .unwrap(); + + let pinned = PinnedRun { job_id: v2_job, script_path: PATH.to_string(), script_hash: V2 }; + let res = asset_graph_for( + &outsider(), + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(pinned), + ) + .await + .unwrap(); + let body = serde_json::to_value(&res.0).unwrap().to_string(); + + assert!( + body.contains("orders"), + "the run they were given still renders its models: {body}" + ); + assert!( + !body.contains("select 2"), + "but not the SQL of a version their grant never covered: {body}" + ); +} diff --git a/backend/windmill-api-assets/tests/fixtures/base.sql b/backend/windmill-api-assets/tests/fixtures/base.sql new file mode 100644 index 0000000000..412fa1029f --- /dev/null +++ b/backend/windmill-api-assets/tests/fixtures/base.sql @@ -0,0 +1,146 @@ +-- used for backend automated testing +-- https://docs.rs/sqlx/latest/sqlx/attr.test.html + +INSERT INTO workspace + (id, name, owner) + VALUES ('test-workspace', 'test-workspace', 'test-user'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test@windmill.dev', 'test-user', true, 'Admin'); + +INSERT INTO workspace_key(workspace_id, kind, key) VALUES + ('test-workspace', 'cloud', 'test-key'); + + +INSERT INTO workspace_settings (workspace_id) VALUES + ('test-workspace'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('test@windmill.dev', 'not-a-real-hash', 'password', true, true, 'Test User', 'test-user'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('test2@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 2'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('test3@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 3'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test2@windmill.dev', 'test-user-2', false, 'User'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test3@windmill.dev', 'test-user-3', false, 'User'); + +insert INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('SECRET_TOKEN'::bytea), 'hex'), 'SECRET_TOK', 'SECRET_TOKEN', 'test@windmill.dev', 'test token', true); +insert INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('SECRET_TOKEN_2'::bytea), 'hex'), 'SECRET_TOK', 'SECRET_TOKEN_2', 'test2@windmill.dev', 'test token 2', false); +insert INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('SECRET_TOKEN_3'::bytea), 'hex'), 'SECRET_TOK', 'SECRET_TOKEN_3', 'test3@windmill.dev', 'test token 3', false); + +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin; +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; + +CREATE FUNCTION "notify_insert_on_completed_job" () +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('completed', NEW.id::text); + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + CREATE TRIGGER "notify_insert_on_completed_job" + AFTER INSERT ON "v2_job_completed" + FOR EACH ROW +EXECUTE FUNCTION "notify_insert_on_completed_job" (); + + +CREATE FUNCTION "notify_queue" () +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('queued', NEW.id::text); + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + CREATE TRIGGER "notify_queue_after_insert" + AFTER INSERT ON "v2_job_queue" + FOR EACH ROW +EXECUTE FUNCTION "notify_queue" (); + + CREATE TRIGGER "notify_queue_after_flow_status_update" + AFTER UPDATE ON "v2_job_status" + FOR EACH ROW + WHEN (NEW.flow_status IS DISTINCT FROM OLD.flow_status) +EXECUTE FUNCTION "notify_queue" (); + +-- Apply phase 4: +DROP FUNCTION IF EXISTS v2_job_after_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_completed_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_completed_before_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_queue_after_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_queue_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_queue_before_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_runtime_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_runtime_before_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_status_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_status_before_update CASCADE; + +DROP VIEW IF EXISTS completed_job, completed_job_view, job, queue, queue_view CASCADE; + +ALTER TABLE v2_job_queue + DROP COLUMN IF EXISTS __parent_job CASCADE, + DROP COLUMN IF EXISTS __created_by CASCADE, + DROP COLUMN IF EXISTS __script_hash CASCADE, + DROP COLUMN IF EXISTS __script_path CASCADE, + DROP COLUMN IF EXISTS __args CASCADE, + DROP COLUMN IF EXISTS __logs CASCADE, + DROP COLUMN IF EXISTS __raw_code CASCADE, + DROP COLUMN IF EXISTS __canceled CASCADE, + DROP COLUMN IF EXISTS __last_ping CASCADE, + DROP COLUMN IF EXISTS __job_kind CASCADE, + DROP COLUMN IF EXISTS __env_id CASCADE, + DROP COLUMN IF EXISTS __schedule_path CASCADE, + DROP COLUMN IF EXISTS __permissioned_as CASCADE, + DROP COLUMN IF EXISTS __flow_status CASCADE, + DROP COLUMN IF EXISTS __raw_flow CASCADE, + DROP COLUMN IF EXISTS __is_flow_step CASCADE, + DROP COLUMN IF EXISTS __language CASCADE, + DROP COLUMN IF EXISTS __same_worker CASCADE, + DROP COLUMN IF EXISTS __raw_lock CASCADE, + DROP COLUMN IF EXISTS __pre_run_error CASCADE, + DROP COLUMN IF EXISTS __email CASCADE, + DROP COLUMN IF EXISTS __visible_to_owner CASCADE, + DROP COLUMN IF EXISTS __mem_peak CASCADE, + DROP COLUMN IF EXISTS __root_job CASCADE, + DROP COLUMN IF EXISTS __leaf_jobs CASCADE, + DROP COLUMN IF EXISTS __concurrent_limit CASCADE, + DROP COLUMN IF EXISTS __concurrency_time_window_s CASCADE, + DROP COLUMN IF EXISTS __timeout CASCADE, + DROP COLUMN IF EXISTS __flow_step_id CASCADE, + DROP COLUMN IF EXISTS __cache_ttl CASCADE; + +LOCK TABLE v2_job_queue IN ACCESS EXCLUSIVE MODE; +ALTER TABLE v2_job_completed + DROP COLUMN IF EXISTS __parent_job CASCADE, + DROP COLUMN IF EXISTS __created_by CASCADE, + DROP COLUMN IF EXISTS __created_at CASCADE, + DROP COLUMN IF EXISTS __success CASCADE, + DROP COLUMN IF EXISTS __script_hash CASCADE, + DROP COLUMN IF EXISTS __script_path CASCADE, + DROP COLUMN IF EXISTS __args CASCADE, + DROP COLUMN IF EXISTS __logs CASCADE, + DROP COLUMN IF EXISTS __raw_code CASCADE, + DROP COLUMN IF EXISTS __canceled CASCADE, + DROP COLUMN IF EXISTS __job_kind CASCADE, + DROP COLUMN IF EXISTS __env_id CASCADE, + DROP COLUMN IF EXISTS __schedule_path CASCADE, + DROP COLUMN IF EXISTS __permissioned_as CASCADE, + DROP COLUMN IF EXISTS __raw_flow CASCADE, + DROP COLUMN IF EXISTS __is_flow_step CASCADE, + DROP COLUMN IF EXISTS __language CASCADE, + DROP COLUMN IF EXISTS __is_skipped CASCADE, + DROP COLUMN IF EXISTS __raw_lock CASCADE, + DROP COLUMN IF EXISTS __email CASCADE, + DROP COLUMN IF EXISTS __visible_to_owner CASCADE, + DROP COLUMN IF EXISTS __tag CASCADE, + DROP COLUMN IF EXISTS __priority CASCADE; diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index cc7fdcb167..d92c3a5f2d 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -1338,7 +1338,11 @@ mod tests { // A label-less token: the job WM_TOKEN, and any token created without one. None, ] { - assert_eq!(kind_of(label).as_deref(), Some("webhook"), "label {label:?}"); + assert_eq!( + kind_of(label).as_deref(), + Some("webhook"), + "label {label:?}" + ); } } @@ -1347,8 +1351,10 @@ mod tests { #[test] fn a_real_trigger_wins_over_the_token_fallback() { let authed = ApiAuthed::default(); - let schedule = - TriggerMetadata::new(Some("u/alice/nightly".to_string()), JobTriggerKind::Schedule); + let schedule = TriggerMetadata::new( + Some("u/alice/nightly".to_string()), + JobTriggerKind::Schedule, + ); let kept = authed.trigger_or_fallback(Some(schedule)).unwrap(); assert_eq!(kept.trigger_kind.to_string(), "schedule"); diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 851dcbb4cd..e9eeb88996 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -964,3 +964,46 @@ async fn test_get_imports(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// A dbt project names a warehouse and nothing else, so the workspace setting is +/// the only place the connection is decided. Two things have to hold for that to +/// work at all: the setting round-trips as the MAP the resolver reads (an +/// envelope stored verbatim makes every warehouse name unresolvable), and the +/// route that serves the name to a worker with no database stays job-scoped. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_warehouses(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{base}/workspaces/edit_dbt_warehouses"))) + .json(&json!({ + "dbt_warehouses": { "main": { "resource_path": "u/admin/wh", "target": "prod" } } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed(client().get(format!("{base}/workspaces/get_settings"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let settings = resp.json::().await?; + assert_eq!( + settings["dbt_warehouses"], + json!({ "main": { "resource_path": "u/admin/wh", "target": "prod" } }) + ); + + // A user token is not a job token: the warehouses a workspace configures are + // a running job's business, not a browsable list. + let resp = authed(client().get(format!("{base}/dbt/warehouse/main"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + + Ok(()) +} diff --git a/backend/windmill-api-scripts/src/asset_inference.rs b/backend/windmill-api-scripts/src/asset_inference.rs index 5debb0d271..9c503ae1ae 100644 --- a/backend/windmill-api-scripts/src/asset_inference.rs +++ b/backend/windmill-api-scripts/src/asset_inference.rs @@ -42,6 +42,13 @@ fn parse_assets_for_lang( #[cfg(feature = "python")] ScriptLang::Python3 => windmill_parser_py_asset::parse_assets(content), ScriptLang::Ansible => windmill_parser_yaml::parse_assets(content), + // dbt is the one language whose assets are not a function of the script + // content: they come from `manifest.json`, which needs a clone of the + // project's repo and a dbt invocation. The deploy-time dependency job + // does that parse and writes the `asset` rows itself + // (`dbt_executor::dbt_dep`), so returning None here is what keeps this + // path from clobbering them with an empty list. + ScriptLang::Dbt => return None, _ => return None, }; match parsed { diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 57e2cef3c3..2d4f1c72b8 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -804,7 +804,20 @@ async fn is_noop_deploy_against_parent( if content != &parent.content { return Ok(false); } - if normalize_optional_text(lock.as_deref()) != normalize_optional_text(parent.lock.as_deref()) { + // A dbt lock is DERIVED, never supplied: the request carries none — the CLI + // sends `lock: undefined` and this route discards any anyway — while a + // deployed parent carries what its dependency job wrote. Comparing them makes + // every unchanged push a new version, a fresh dependency job and the sync + // activity `skip_if_noop` exists to prevent. The parent must actually hold + // one, or a deploy whose dependency job failed could never be retried by + // pushing the same project again. + if matches!(language, ScriptLang::Dbt) { + if normalize_optional_text(parent.lock.as_deref()).is_empty() { + return Ok(false); + } + } else if normalize_optional_text(lock.as_deref()) + != normalize_optional_text(parent.lock.as_deref()) + { return Ok(false); } if summary != &parent.summary { @@ -853,7 +866,22 @@ async fn is_noop_deploy_against_parent( if on_behalf_of_email != &parent.on_behalf_of_email { return Ok(false); } - if !schema_opt_eq(schema.as_ref(), parent.schema.as_ref()) { + // Both of a dbt script's derived fields are compared as they WOULD BE STORED, + // not as they arrived: the schema comes from the descriptor and the clients + // cannot derive it (`windmill-parser-wasm` has no dbt arm), so they send the + // previous version's or none at all. Comparing what they sent makes every + // unchanged push differ. + let dbt_schema = matches!(language, ScriptLang::Dbt) + .then(|| windmill_parser_yaml::dbt_arg_schema(content).ok()) + .flatten() + .and_then(|v| serde_json::value::to_raw_value(&v).ok()) + .map(|v| Schema(sqlx::types::Json(v))); + let effective_schema = if matches!(language, ScriptLang::Dbt) { + dbt_schema.as_ref() + } else { + schema.as_ref() + }; + if !schema_opt_eq(effective_schema, parent.schema.as_ref()) { return Ok(false); } if !json_serialize_eq(assets, &parent.assets) { @@ -1185,6 +1213,25 @@ async fn create_script_internal<'c>( .as_ref() .map(|v| v.perms.clone()) .unwrap_or(json!({})); + // A dbt lock names a manifest digest and engine versions that only a + // dependency job can determine, and that job is also what publishes the + // script's manifest graph. Honouring a supplied one would skip it, leaving + // the script with no graph — including on the UI paths that round-trip an + // existing lock, like rename and unarchive. + if matches!(ns.language, ScriptLang::Dbt) { + ns.lock = None; + // A codebase is a bundle of JS/TS sources; a dbt script's project is + // its modules. Accepting one takes the branch below that stands in for + // lock generation, which would suppress the very job that parses the + // project and publishes the graph. + if ns.codebase.is_some() { + return Err(Error::BadRequest( + "a dbt script has no codebase: its project is its modules, the \ + ` @@ -712,6 +721,10 @@ {/if} +{#if dbtRun} + +{/if} + {#if result_stream && result == undefined}
diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index 73c9a08c4a..bb51851003 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -26,7 +26,10 @@ interface Props { code?: string - language: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined + // `sql` is the dialect-agnostic option: a dbt model's SQL is compiled by + // whichever adapter the project targets, so naming one dialect would be a + // guess. Every dialect below highlights through the same grammar anyway. + language: Script['language'] | 'bunnative' | 'frontend' | 'json' | 'sql' | undefined highlightLanguage?: LanguageType | undefined lines?: boolean className?: string @@ -56,7 +59,9 @@ ? 'opacity-0 group-hover:opacity-100 focus-within:opacity-100 transition-opacity duration-150' : '' - function getLang(lang: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined) { + function getLang( + lang: Script['language'] | 'bunnative' | 'frontend' | 'json' | 'sql' | undefined + ) { switch (lang) { case 'python3': return python @@ -76,6 +81,8 @@ return javascript case 'graphql': return graphql + case 'sql': + return sql case 'mysql': return sql case 'postgresql': diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index bd30cd6909..f209907537 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -454,6 +454,8 @@ language: 'bun' } } + } else if (script.language === 'dbt') { + seedDbtProject() } const restarter = scheduleRestartSync(userDraftPath, { waitForContent: true }) initContent(script.language, script.kind, template).finally(() => restarter.markContentReady()) @@ -971,6 +973,34 @@ function handleDeployTrigger(_trigger: Trigger) {} + // A dbt script's modules ARE its dbt project, and the runtime refuses one + // without a `dbt_project.yml`. Seeded from BOTH entry points — the empty-script + // bootstrap and the language picker — because reaching dbt by switching an + // existing draft otherwise produces a script that cannot deploy or run. + // Existing modules are left alone: switching away and back must not discard a + // project the user has already grown. + function seedDbtProject() { + // Keyed on the project file rather than on "has any modules at all": a + // draft that grew modules under another language carries none of what dbt + // needs, and the worker refuses a bundle with no `dbt_project.yml` — so + // that draft reached dbt in a state it could neither deploy nor run. + if (script.modules?.['dbt_project.yml']) return + script.modules = { + 'dbt_project.yml': { + content: + 'name: my_dbt_project\nversion: "1.0"\nprofile: my_dbt_project\nmodels:\n my_dbt_project:\n +materialized: view\n', + language: 'dbt' + }, + 'models/example.sql': { + content: 'select 1 as id\n', + language: 'dbt' + }, + // Last, so anything already written wins: the previous language's + // helper files are inert to dbt and are the user's to remove. + ...(script.modules ?? {}) + } + } + function onScriptLanguageTrigger(lang: 'docker' | 'bunnative' | ScriptLang) { if (lang == 'docker') { template = 'docker' @@ -983,6 +1013,9 @@ // initContent(language, script.kind, template) script.language = language + if (language === 'dbt') { + seedDbtProject() + } } function onSummaryChange(value: string) { diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 9a65fc9d2b..e112f92b3b 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -19,6 +19,10 @@ import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow' import WacDiagram from '$lib/components/graph/WacDiagram.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' + import DbtProjectPanel, { + dbtFileLang, + dbtModelSelector + } from '$lib/components/dbt/DbtProjectPanel.svelte' import SchemaForm from './SchemaForm.svelte' import PowerShellCommonParams from './PowerShellCommonParams.svelte' import LogPanel from './scriptEditor/LogPanel.svelte' @@ -354,9 +358,32 @@ editor?.setCode(editorCode) } + // Whether the open file is tested as a runnable of its own. A `__mod` helper + // is; a dbt project's files are not — the run is always the project's, so the + // arguments shown, edited and logged must be the descriptor's, not an empty + // per-module set the request would ignore. + let onModuleArgs = $derived(activeModuleTab !== null && lang !== 'dbt') + + // The selector a Test would build with, when the open file is a model. Macros, + // analyses and singular tests are `.sql` too and none is selectable by name, + // so those fall back to running the project. + let dbtSelected = $derived.by(() => { + const open = activeModuleTab + if (lang !== 'dbt' || !open) return undefined + const selector = dbtModelSelector(modules ?? {}, open) + // The label drops whichever extension the selector matched, so a Python + // model reads `Build my_model` rather than `Build my_model.py`. + const name = open.split('/').pop()!.replace(/\.(sql|py)$/, '') + return selector ? { selector, name } : undefined + }) + let effectiveLang = $derived( activeModuleTab && modules?.[activeModuleTab] - ? (modules[activeModuleTab].language as Preview['language']) + ? lang === 'dbt' + ? // Every dbt module is stored as `dbt`; the extension is what says + // whether this file is SQL, YAML or a seed. + dbtFileLang(activeModuleTab) + : (modules[activeModuleTab].language as Preview['language']) : lang ) @@ -373,7 +400,15 @@ return isTsWac || isPyWac }) let supportsModules = $derived((lang === 'bun' || lang === 'python3') && isWacV2) - let mainFileName = $derived('script.' + langToExt(scriptLangToEditorLang(lang))) + // A dbt script's content is the descriptor and its modules are the project. + // A tree rather than the module tab strip: a project has folders and dozens + // of files, which a strip cannot show. + let isDbt = $derived(lang === 'dbt') + let mainFileName = $derived( + isDbt + ? 'wm_dbt.yaml' + : 'script.' + langToExt(scriptLangToEditorLang(lang)) + ) let modulePathInput = $state('') let showAddModulePopover = $state(false) @@ -428,13 +463,28 @@ bunnative: ['.ts'] } + // A dbt project's files are dbt's own, not Windmill modules: models and tests + // are `.sql`, schemas and the project file `.yml`, seeds `.csv`, docs `.md`. + // `.py` because dbt Python models are first-class on Snowflake, BigQuery and + // Databricks, and the CLI already bundles one; refusing to CREATE one here + // was the only place that restriction existed. + const DBT_MODULE_EXTENSIONS = ['.sql', '.py', '.yml', '.yaml', '.csv', '.md'] let allowedModuleExtensions = $derived( - lang - ? (LANG_MODULE_EXTENSIONS[lang] ?? Object.keys(ALL_MODULE_EXTENSIONS)) - : Object.keys(ALL_MODULE_EXTENSIONS) + lang === 'dbt' + ? DBT_MODULE_EXTENSIONS + : lang + ? (LANG_MODULE_EXTENSIONS[lang] ?? Object.keys(ALL_MODULE_EXTENSIONS)) + : Object.keys(ALL_MODULE_EXTENSIONS) ) function inferModuleLang(filePath: string): ScriptModule['language'] | undefined { + // Every file of a dbt project is stored as `dbt`, whatever its extension: + // they are the project's, and dbt is what reads them. + if (lang === 'dbt') { + return DBT_MODULE_EXTENSIONS.some((e) => filePath.endsWith(e)) + ? ('dbt' as ScriptModule['language']) + : undefined + } for (const [ext, moduleLang] of Object.entries(ALL_MODULE_EXTENSIONS)) { if (filePath.endsWith(ext)) return moduleLang } @@ -442,6 +492,12 @@ } function getModuleDefaultContent(filePath: string): string { + if (lang === 'dbt') { + // A model that compiles on its own, so a new file is runnable before it + // is edited; anything else starts empty rather than with a guess at + // which dbt schema it is. + return filePath.endsWith('.sql') ? `select 1 as id\n` : '' + } if (filePath.endsWith('.py')) { return `def hello() -> str:\n return "world"\n` } else if (filePath.endsWith('.ts')) { @@ -474,8 +530,19 @@ return '' } + /// The descriptor is the script's CONTENT, not a module. A module at that same + /// path would be a second, independent value for one file: the export writes + /// the content there, and the bundle would emit over it. + function reservedDbtPath(path: string): string | undefined { + return lang === 'dbt' && path.trim() === 'wm_dbt.yaml' + ? `wm_dbt.yaml is the descriptor, edited from the tree — it cannot also be a file` + : undefined + } + function validateModulePath(path: string): string { if (!path.trim()) return '' + const reserved = reservedDbtPath(path) + if (reserved) return reserved const moduleLang = inferModuleLang(path) if (!moduleLang) { const exts = allowedModuleExtensions.join(', ') @@ -525,6 +592,8 @@ function validateRenameModulePath(newPath: string, oldPath: string): string { if (!newPath.trim()) return '' + const reserved = reservedDbtPath(newPath) + if (reserved) return reserved const moduleLang = inferModuleLang(newPath) if (!moduleLang) { const exts = allowedModuleExtensions.join(', ') @@ -838,16 +907,30 @@ // Flush module edits back to modules map before running preview flushModuleContent() - const testCode = activeModuleTab !== null ? editorCode : code - const testLang = activeModuleTab !== null ? effectiveLang : lang - const rawTestArgs = - activeModuleTab !== null - ? testPanelArgs - : selectedTab === 'preprocessor' || kind === 'preprocessor' - ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...(args ?? {}) } - : (args ?? {}) - const testSchema = activeModuleTab !== null ? testPanelSchema : schema + // A dbt run is always the project's, whichever file is open: `dbt build` + // takes the whole bundle, and testing one model in isolation is not a + // thing dbt does. + const onModule = onModuleArgs + const testCode = onModule ? editorCode : code + const testLang = onModule ? effectiveLang : lang + const rawTestArgs = onModule + ? testPanelArgs + : selectedTab === 'preprocessor' || kind === 'preprocessor' + ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...(args ?? {}) } + : (args ?? {}) + const testSchema = onModule ? testPanelSchema : schema const testArgs = await processSecretArgs(rawTestArgs, testSchema, opWs) + // Testing with a model open builds THAT model: `dbt build --select ` + // is dbt's own inner loop, and running the whole project to check one file + // is the thing a dbt developer never does. Its tests come along, because + // `build` interleaves them. + if (dbtSelected) { + testArgs.command = { + ...((testArgs.command as object) ?? {}), + label: 'build', + select: [dbtSelected.selector] + } + } if (showPsCommonParams) { for (const [k, v] of Object.entries(psCommonParams)) { if (v !== undefined && v !== false && v !== '') { @@ -891,7 +974,10 @@ } }, undefined, - activeModuleTab !== null ? undefined : modules, + // A `__mod` helper is tested alone, so its siblings are left out. A dbt + // project cannot be: the bundle IS the project, and without it the run + // finds no `dbt_project.yml` whichever file happens to be open. + onModule ? undefined : modules, undefined, timeout ) @@ -1041,6 +1127,11 @@ async function inferModuleSchema() { if (activeModuleTab === null) return + // A dbt project's files are not independently runnable: a model is SQL dbt + // compiles, not a script with arguments. Inferring some would put another + // language's parameters (a `.sql` model reads as Postgres) in the run form + // beside the descriptor's own. + if (lang === 'dbt') return try { await inferArgs(effectiveLang, editorCode, testPanelSchema) injectPartitionArg(testPanelSchema, testPanelArgs, effectiveLang, editorCode) @@ -2241,7 +2332,7 @@ { if (e.detail) { - if (activeModuleTab !== null) { + if (onModuleArgs) { testPanelArgs = e.detail } else { args = e.detail @@ -2259,7 +2350,7 @@ bind:clientHeight={schemaHeight} > {#key argsRender} - {#if activeModuleTab !== null} + {#if onModuleArgs} - Test + + {dbtSelected ? `Build ${dbtSelected.name}` : 'Test'} {/snippet} @@ -2371,7 +2464,7 @@ previewIsLoading={debugMode ? $debugState.running && !$debugState.stopped : testIsLoading} {editor} {diffEditor} - args={activeModuleTab !== null ? testPanelArgs : args} + args={onModuleArgs ? testPanelArgs : args} {showCaptures} customUi={customUi?.previewPanel} showCustomResultPanel={showDebugPanel} @@ -2505,7 +2598,34 @@ {/snippet} {#snippet editorContent()} -
+
+ {#if isDbt} + (p === null ? switchToMain() : switchToModule(p))} + onDelete={removeModule} + > + {#snippet addFile()} + + {#snippet trigger()} +
+ +
+ {/snippet} + {#snippet content({ close })} + {@render addModuleForm(close)} + {/snippet} + + {/snippet} + + {/if} {#if supportsModules}
{/if} -
+
{#if assets?.length} diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte index edaefe947a..214ab3e31d 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte @@ -7,7 +7,7 @@ import { inferArgs } from '$lib/infer' import { initialCode } from '$lib/script_helpers' import { emptySchema } from '$lib/utils' - import { defaultScriptLanguages, getScriptByPath, processLangs } from '$lib/scripts' + import { defaultScriptLanguages, getScriptByPath, processInlineLangs } from '$lib/scripts' import { Building, GitFork, Globe2 } from 'lucide-svelte' import { createEventDispatcher } from 'svelte' @@ -88,7 +88,7 @@ } let langs = $derived( - processLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) + processInlineLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) .map((l) => [defaultScriptLanguages[l], l]) .filter( (x) => diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 3e5e9d8cef..8a893e3e84 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -18,10 +18,15 @@ import PanToNode from './PanToNode.svelte' import InitialFitView from './InitialFitView.svelte' import { layoutAssetGraph } from './assetGraphLayout' - import { computeMutedReadKeys } from './resolveGraph' + import { computeMutedReadKeys, dbtAssociations } from './resolveGraph' import { buildDownstreamMap } from './graphTraversal' import { buildLineageDownstreamMap } from './boundedCascade' - import type { AssetGraphResponse, AssetGraphSelection, NativeTriggerKind } from './types' + import type { + AssetGraphResponse, + AssetGraphSelection, + AssetRunState, + NativeTriggerKind + } from './types' import type { RunnableRunState } from './activeRunnables.svelte' import type { AssetKind } from '$lib/gen' import { NODE } from '$lib/components/graph/util' @@ -179,6 +184,11 @@ * When a node's nonce changes, it flashes a fading green background — its * producer just recomputed it. Driven by the replay player frame-by-frame. */ recomputedAssetIds?: ReadonlyMap + /** What a run is currently doing to each relation, keyed `asset::` + * like every other per-asset map here. + * Distinct from `recomputedAssetIds`, which is a one-shot pulse: this is + * the state a node holds until the run moves it. */ + assetRunStatus?: ReadonlyMap /** Let the wheel zoom the canvas (and swallow the page scroll while doing * so). Default true for the full-height editor/player. Set false when the * canvas is embedded inline inside a scrollable container, so a wheel @@ -215,6 +225,7 @@ viewportFitKey = '', highlightActiveRun = false, recomputedAssetIds, + assetRunStatus, scrollZoom = true }: Props = $props() @@ -244,6 +255,7 @@ | 'data-test' | 'macro' | 'test-dependency' + | 'dbt-ref' unsaved?: boolean // Muted read edge: a ducklake/s3 input read every run whose (default) // auto cascade trigger is suppressed by `// mute` / `// mute all`. @@ -287,6 +299,26 @@ // by node id across producers). const addedTestNodes = new Set() + // A dbt script owns every relation its project materializes. Drawing that + // as one edge per model buries the lineage that matters — `ref()` between + // models, and native consumers — under a fan-out that grows with the + // project, so the association is carried by the model's badge and its + // hover/click highlight instead. Only the DRAWING is dropped: the + // producer rows still drive cascade dispatch and "who produced this". + const dbtRunnableIds = new Set( + g.runnables.filter((r) => r.dbt).map((r) => `${r.usage_kind}:${r.path}`) + ) + // Association only — the canvas deliberately draws no edge for it. + const { ownerByAsset: dbtOwnerByAsset, writesByOwner: dbtWritesByOwner } = dbtAssociations( + g.runnables, + g.edges + ) + // Per-relation dbt description, to tell a project's own declared source + // from a relation another script materializes. + const dbtAssetProvenance = new Map( + g.assets.filter((a) => a.dbt).map((a) => [`asset:${a.kind}:${a.path}`, a.dbt!]) + ) + const hasAddNode = onAddPipelineScript != null if (hasAddNode) { nodes.push({ @@ -395,6 +427,7 @@ path: a.path, fork_materialization: a.fork_materialization, derived_from: a.derived_from, + dbt: a.dbt, onAddScript: onAddScriptForAsset, pathPrefix, defaultPathSuffix, @@ -404,7 +437,32 @@ producerFailed, // Bumped by the replay player when this asset's producer just // recomputed it — the node flashes green and fades. - recomputePulse: recomputedAssetIds?.get(assetId) + recomputePulse: recomputedAssetIds?.get(assetId), + // What the run in view is doing to this relation right now. + runStatus: assetRunStatus?.get(assetId)?.status, + runRowCount: assetRunStatus?.get(assetId)?.rowCount, + // The dbt project that materializes this relation, related by + // badge rather than by an edge — so only when that node is on + // this graph. The run page and the pipeline page both hide it, + // and passing handlers anyway makes the chip advertise a click + // that resolves to nothing. + ...(dbtOwnerByAsset.has(assetId) + ? { + onDbtHover: (on: boolean) => (dbtHoverId = on ? assetId : undefined), + onDbtSelect: () => { + // `runnable::` — the id shape `build` uses. + const owner = model.dbtOwnerByAsset.get(assetId) + const [kind, ...rest] = owner?.split(':') ?? [] + if (kind && rest.length) { + onselect?.({ + kind: 'runnable', + runnable_kind: kind as 'script' | 'flow', + path: rest.join(':') + }) + } + } + } + : {}) } }) } @@ -475,6 +533,8 @@ tag: r.tag, retry: r.retry, macros: r.macros, + dbt: r.dbt, + onDbtHover: (on: boolean) => (dbtHoverId = on ? rid : undefined), unsaved: r.unsaved ?? false, // Same dispatch the asset node uses, only routed when the // runnable is a script (the page handler short-circuits @@ -522,10 +582,28 @@ // (`// mute` / `// mute all` opted the default auto trigger out). Gated // on pipeline scripts inside the helper (non-pipeline reads never derive). const mutedReadKeys = computeMutedReadKeys(g.edges, g.triggers, g.runnables) + // A dbt script owns every relation of its project. Drawing that as one + // edge per model buries the lineage that matters (`ref()` between models, + // and native consumers) under a fan-out that grows with the project — so + // the association is carried by the node badge and its hover/click + // highlight instead. Only the DRAWING is dropped: the producer rows still + // drive cascade dispatch and "who produced this". for (const e of g.edges) { const runnableId = `${e.runnable_kind}:${e.runnable_path}` const assetId = `asset:${e.asset_kind}:${e.asset_path}` const access = e.access_type ?? 'r' + // A dbt project's own relations are related by badge, not by edges: its + // writes are the fan-out, and its declared sources already reach its + // models through the `ref()` edges, so both would be noise. + // + // A read of a relation ANOTHER script builds is different — that is how + // two selections of one project compose (decision 6), it carries the + // cascade, and no `ref()` edge survives the split to stand in for it. + // Kept, or the two halves render as disconnected islands. + if (dbtRunnableIds.has(runnableId)) { + const isOwnSource = dbtAssetProvenance.get(assetId)?.resource_type === 'source' + if (access === 'w' || access === 'rw' || isOwnSource) continue + } if (access === 'w' || access === 'rw') { // Data tests assert on the `// materialize` target, which is always // a ducklake asset (v1 enforces this), so only the ducklake @@ -621,6 +699,17 @@ }) } + // dbt `ref()` lineage: model → model inside one project. The dbt script + // writes every one of them, so without these the canvas shows a flat + // fan-out from the script and loses the project's actual shape. + const assetNodeIds = new Set(g.assets.map((a) => `asset:${a.kind}:${a.path}`)) + for (const de of g.dbt_edges ?? []) { + const from = `asset:dbt:${de.from_asset_path}` + const to = `asset:dbt:${de.to_asset_path}` + if (!assetNodeIds.has(from) || !assetNodeIds.has(to)) continue + edges.push({ id: `dbtref:${from}->${to}`, source: from, target: to, kind: 'dbt-ref' }) + } + // Non-asset triggers (schedule + native) are rendered as source nodes // above the pipeline script. Real (non-missing) nodes are // deduplicated per (kind, ref) tuple so a single schedule shared @@ -787,11 +876,24 @@ } } - return { nodes, edges } + return { nodes, edges, dbtOwnerByAsset, dbtWritesByOwner } } let model = $derived(build(graph)) + // dbt association, surfaced by emphasis instead of edges. Hovering a model's + // dbt badge lights up the project node that materializes it; hovering the + // project node lights up every model it owns. Clicking the badge selects the + // project node, so the association survives the pointer leaving. + let dbtHoverId = $state(undefined) + let dbtEmphasisIds = $derived.by(() => { + if (!dbtHoverId) return new Set() + const owned = model.dbtWritesByOwner.get(dbtHoverId) + if (owned) return new Set([dbtHoverId, ...owned]) + const owner = model.dbtOwnerByAsset.get(dbtHoverId) + return owner ? new Set([dbtHoverId, owner]) : new Set() + }) + let selectedId = $derived.by(() => { if (!selection) return undefined return selection.kind === 'asset' @@ -895,12 +997,13 @@ else if (boundPick.bounded.has(n.id)) boundClass = 'wm-bound-in' else if (!boundPick.eligible.has(n.id)) boundClass = 'wm-bound-dim' } + const dbtClass = dbtEmphasisIds.has(n.id) ? 'wm-dbt-linked' : undefined return { id: n.id, type: n.type, position: { x: p.x + xCenter + xShift, y: p.y + 40 }, data: n.data, - class: boundClass ?? runClass ?? assetClass, + class: boundClass ?? dbtClass ?? runClass ?? assetClass, selected: n.id === selectedId, // All nodes non-draggable: the layout is sugiyama-computed, // dragging would fight the reactive re-layout. Selection is @@ -1064,6 +1167,21 @@ label = 'test needs' labelStyle = 'fill: rgb(217 119 6); font-size: 10px; font-weight: 600;' break + case 'dbt-ref': + // model → model inside one dbt project. Orange, matching the + // dbt badges, and dashed because the edge is dbt's own lineage + // rather than a Windmill read/write the cascade acts on. + style = 'stroke: rgb(234 88 12); stroke-width: 1.25px;' + strokeDasharray = '4 3' + markerColor = 'rgb(234 88 12)' + label = 'ref' + labelStyle = 'fill: rgb(234 88 12); font-size: 10px; font-weight: 600;' + // Same rule the pipeline uses for a running script: the edges + // touching what is happening animate. Here the unit of work is + // the model, so the edges feeding the one dbt is building move, + // and the flow reads in DAG order as it advances. + animated = assetRunStatus?.get(e.target)?.status === 'running' + break default: style = '' } @@ -1254,6 +1372,11 @@ /* Activity-panel emphasis — soft, monochromatic, less prominent than the blue details selection above. Hover is a thin neutral ring (transient); pinning an expanded run is a soft-blue ring. */ + /* A dbt project node and the models it materializes, related by badge + rather than by edges — hovering either lights up the whole set. */ + :global(.svelte-flow__node.wm-dbt-linked .drop-shadow-sm) { + @apply outline outline-2 outline-orange-400/80; + } :global(.svelte-flow__node.wm-run-hover .drop-shadow-sm) { @apply outline outline-1 outline-gray-400 dark:outline-gray-500; } diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index d48ecbe09e..2c5e5fd97e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -27,7 +27,9 @@ import { inferArgs } from '$lib/infer' import { emptySchema, sendUserToast } from '$lib/utils' import type { Schema } from '$lib/common' - import type { AssetGraphSelection, PipelineMode } from './types' + import type { AssetGraphSelection, DbtAssetProvenance, PipelineMode } from './types' + import HighlightCode from '$lib/components/HighlightCode.svelte' + import DbtIcon from '$lib/components/icons/DbtIcon.svelte' import PipelineScriptView from './PipelineScriptView.svelte' import { parsePipelineAnnotations, @@ -154,6 +156,9 @@ // resolved graph). Drives the transitive column-lineage trace shown for a // selected materialized asset. selectionColumnGraph?: ColumnLineageGraph + /** dbt provenance of the selected relation, when a dbt project + * materializes it — carries the model's own SQL. */ + selectionDbt?: DbtAssetProvenance // Whether the selected ducklake asset's schema can evolve (whole-table // `replace` producer). Forwarded to the Schema tab: version history when // true, a single fixed-schema view when false. Defaults to true (unknown). @@ -284,6 +289,7 @@ onScriptRemoved, selectionProducers = [], selectionColumnGraph, + selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, schemaContractContext = undefined, @@ -426,6 +432,20 @@ ) ) + // Where the selected model's file sits on disk: the producing script's + // module folder holds the dbt project verbatim, so this is the path a + // `wmill sync pull` writes and the one to edit. + let dbtBundlePath = $derived.by(() => { + const file = selectionDbt?.original_file_path + if (!file) return undefined + // A relation may have several script producers, and nothing here says which + // of them is the dbt project this model came from. Prefixing the wrong one + // names a `__dbt` folder that does not exist, so an ambiguous relation shows + // the path inside the project alone. + const scripts = selectionProducers.filter((p) => p.kind === 'script') + return scripts.length === 1 ? `${scripts[0].path}__dbt/${file}` : file + }) + // Bound from ScriptEditor — populated by inferAssets on every code // change. Forwarded to the page so the canvas can re-derive write // edges as the user edits the body (e.g. renaming a CREATE TABLE @@ -1216,6 +1236,25 @@
{/key} + {:else if selectionDbt?.raw_code} + +
+
+ + {dbtBundlePath ?? selectionDbt.unique_id} + read-only · edit locally +
+
+ +
+
{:else}
No inline preview yet for {selection.asset_kind}. Use the producer/consumer arrows diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte index 999559a0c7..815db36b18 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -14,13 +14,17 @@ Loader2, Plus, ShieldCheck, - ShieldAlert + ShieldAlert, + CheckCircle2, + XCircle } from 'lucide-svelte' import type { ScriptLang } from '$lib/gen' import { enterpriseLicense, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/utils' import { PIPELINE_LANGUAGES } from './pipelineLanguages' import type { PipelineOutputKind } from './pipelineTemplates' + import type { DbtAssetProvenance } from './types' + import DbtIcon from '$lib/components/icons/DbtIcon.svelte' // Shape used for both the data prop and the run callback. Drafts carry // `content` / `language` so the page-level run handler can dispatch to @@ -44,6 +48,15 @@ // "current view of " marker so it reads as a derived node, not an // unrelated table. derived_from?: string + // dbt provenance when this warehouse table is a dbt node: which model + // it is, how dbt materializes it, its tags and its generic tests. + dbt?: DbtAssetProvenance + /** Hovering the dbt chip emphasizes the project node that + * materializes this model — the association the graph deliberately + * does not draw as an edge. */ + onDbtHover?: (on: boolean) => void + /** Clicking it selects that project node. */ + onDbtSelect?: () => void onAddScript?: ( asset: { kind: AssetKind; path: string }, language: ScriptLang, @@ -78,6 +91,11 @@ // producer just recomputed it. A change triggers a one-shot green // fade so a freshly-written table stands out as the run progresses. recomputePulse?: number + // What the run being viewed is doing to this relation. dbt records it + // per model as it walks the DAG, so the graph moves with the run + // instead of only settling once the job ends. + runStatus?: 'running' | 'materialized' | 'failed' + runRowCount?: number | null } // SvelteFlow injects this on the node component when the user clicks // the node. Combined with our own `hovered` state to drive the @@ -133,6 +151,40 @@ let showAdd = $derived(data.onAddScript != undefined) + // dbt badge. `materialized` is dbt's own word rather than the Windmill + // strategy because `view` and `ephemeral` have no strategy, and showing the + // dbt word keeps the node legible to someone reading their own project. + let dbtLabel = $derived(data.dbt?.materialized ?? data.dbt?.resource_type) + let dbtTitle = $derived.by(() => { + const d = data.dbt + if (!d) return '' + const lines = [`dbt ${d.resource_type}: ${d.unique_id}`] + if (d.materialized) { + const strategy = d.materialize_strategy ? ` -> ${d.materialize_strategy}` : '' + lines.push(`materialized: ${d.materialized}${strategy}`) + } + if (d.tags?.length) lines.push(`tags: ${d.tags.join(', ')}`) + for (const t of d.data_tests ?? []) { + const col = t.column ? ` on ${t.column}` : '' + lines.push(`test ${t.kind}${col}${t.severity ? ` [${t.severity}]` : ''}`) + } + const cols = Object.entries(d.columns ?? {}) + if (cols.length) { + lines.push( + `columns: ${cols.map(([c, desc]) => (desc ? `${c} (${desc})` : c)).join(', ')}` + ) + } + if (d.freshness) { + const f = d.freshness as Record + const window = (k: string) => + f[k]?.count != null ? `${k.replace('_after', '')} after ${f[k].count}${f[k].period?.[0] ?? ''}` : '' + const windows = ['warn_after', 'error_after'].map(window).filter(Boolean) + if (windows.length) lines.push(`freshness: ${windows.join(', ')}`) + } + if (d.description) lines.push(d.description) + return lines.join('\n') + }) + // Data-test outcome badge. Only guarded assets show it. The write's fate on a // failing test differs by edition — surface which one applies so a shared // parent/fork table name can't hide a silently-published bad version. @@ -200,6 +252,39 @@ class={`shrink-0 ml-2 mr-2 ${selected ? 'text-accent' : 'text-blue-600 dark:text-blue-400'}`} size="14px" /> + {#if data.runStatus} + + + {#if data.runStatus === 'running'} + + {:else if data.runStatus === 'failed'} + + {:else} + + {/if} + + + {#if data.runStatus === 'materialized' && data.runRowCount != undefined} + + {Intl.NumberFormat().format(data.runRowCount)} + + {/if} + {/if} {formatShortAssetPath(asset)} @@ -225,6 +310,34 @@ fork {/if} + + {#if data.dbt} + + {/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 7db35c2353..ec28b7c784 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -16,8 +16,7 @@ AssetGraphResponse, AssetGraphSelection, NativeTriggerKind, - PipelineMode - } from './types' + PipelineMode, DbtAssetProvenance } from './types' import type { AssetKind, Script, ScriptLang } from '$lib/gen' import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte' import type { PipelineOutputKind } from './pipelineTemplates' @@ -76,6 +75,7 @@ localScriptsVersion, selectionProducers = [], selectionColumnGraph, + selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, schemaContractContext = undefined, @@ -181,6 +181,8 @@ selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> /** Transitive column-lineage trace for a selected ducklake asset (route page). */ selectionColumnGraph?: ColumnLineageGraph + /** dbt provenance of the selected relation — carries its SQL. */ + selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean /** Fork workspaces: data-environment state of the selected ducklake asset (route page). */ selectionForkMaterialization?: 'fork' | 'deferred' @@ -512,6 +514,7 @@ selection={activeDraft ? undefined : editor.selection} selectionProducers={activeDraft ? [] : selectionProducers} {selectionColumnGraph} + {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} {schemaContractContext} diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index d8a977539b..c260bc2e5b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -18,6 +18,7 @@ XCircle, Zap } from 'lucide-svelte' + import DbtIcon from '$lib/components/icons/DbtIcon.svelte' import { twMerge } from 'tailwind-merge' import { preventDefault, stopPropagation } from 'svelte/legacy' import type { GraphUsageKind } from './types' @@ -45,6 +46,13 @@ // Macros this script provides (deployed/drafted `// macros` library). // Non-empty renders the ƒ chip marking the node as a macro library. macros?: { name: string; params: string; is_table: boolean }[] + // Set on a dbt script: the number of models the project materializes. + // One runnable node stands for the whole project, so the count is what + // tells it apart from a single-output script. + dbt?: { model_count: number } + /** Hovering the project badge emphasizes every model it + * materializes — the fan-out the graph deliberately omits. */ + onDbtHover?: (on: boolean) => void // Last-run status + run count observed this session (from the // folder queue poll). Undefined until the first observed run. runState?: RunnableRunState @@ -188,7 +196,19 @@ -
(hover = true)} onmouseleave={() => (hover = false)}> + +
{ + hover = true + data.onDbtHover?.(true) + }} + onmouseleave={() => { + hover = false + data.onDbtHover?.(false) + }} +> + {#if onDelete && node.path !== 'dbt_project.yml'} + + {/if} +
+ {/if} + {/each} +{/snippet} + +
+
+ {scriptPath}__dbt/ +
+ {fileCount + 1} + {@render addFile?.()} +
+
+
+ + + {@render branch(tree, 0)} +
+ {#if fileCount === 0} +
+ No project yet. Copy one in and push it: + 
cp -r my-dbt-project/. {scriptPath}__dbt/
+wmill sync push
+
+ {/if} +
diff --git a/frontend/src/lib/components/dbt/DbtRunGraph.svelte b/frontend/src/lib/components/dbt/DbtRunGraph.svelte new file mode 100644 index 0000000000..7b9cbee07b --- /dev/null +++ b/frontend/src/lib/components/dbt/DbtRunGraph.svelte @@ -0,0 +1,769 @@ + + +{#snippet sqlPane()} + {#if selectedIsForeign} +
+ Another dbt project in this workspace also materializes this relation, and the graph keeps one + project's model per relation — so the SQL shown here would not be this run's. Open that + project's own run to see it. +
+ {:else if selectedDbt?.raw_code} +
+
+ {selectedDbt.original_file_path ?? selectedDbt.unique_id} + {#if selectedDbt.materialized} + {selectedDbt.materialized} + {/if} + + {#if selectedDbt.resource_type === 'model'} + {#if showRows && preview && !('error' in preview)} + + {:else} + + {/if} + {/if} + {#if selectedRelation} + + {/if} + + + read-only · edit in the script + +
+
+ {#if showRows && preview} + {#if 'error' in preview} +
{preview.error}
+ + {:else if 'pending' in preview} +
+ + Running `dbt show` — this is a job, so it waits on a worker and the engine. +
+ {:else} + {@const cols = Object.keys(preview.rows[0] ?? {})} + {#if cols.length === 0} +
The model returned no rows.
+ {:else} + + + + {#each cols as c (c)} + + {/each} + + + + {#each preview.rows as row, i (i)} + + {#each cols as c (c)} + + {/each} + + {/each} + +
{c}
{cellText(row[c])}
+
+ {preview.rows.length} rows in {(preview.tookMs / 1000).toFixed(1)}s{preview.node + ? ` · ${preview.node}` + : ''} +
+ {/if} + {/if} + {:else} + + {/if} +
+
+ {/if} +{/snippet} + +{#if resumable} +
+ {(run?.totals?.error ?? 0) > 0 ? `${run?.totals?.error} failed` : ''}{(run?.totals?.error ?? + 0) > 0 && (run?.totals?.skipped ?? 0) > 0 + ? ', ' + : ''}{(run?.totals?.skipped ?? 0) > 0 ? `${run?.totals?.skipped} skipped` : ''}. Rebuild only + those with dbt retry, instead of the whole project. + + + +
+{/if} + +{#if loading} +
+ Loading the model graph +
+{:else if failed} +
Could not load the model graph.
+{:else if !graph} +
+ {#if ranTestsOnly} + This run selected tests alone, so it built no models. A dbt test is an assertion rather than a + relation, so it has no node here — the models it asserts against belong to the runs that build + them. Its results are in the table below. + {:else} + This dbt script has no models in the asset graph. A project that brings its own + profiles.yml without naming a + profile.warehouse has no warehouse identity to key them on. + {/if} +
+{:else} +
+ {#if relationDrift > 0} +
+ {relationDrift} + {relationDrift === 1 ? 'model has' : 'models have'} been renamed or moved since this run — + {relationDrift === 1 ? 'its node shows' : 'their nodes show'} today's relation, not the one this + run wrote. +
+ {/if} + {#if goneSinceRun > 0} +
+ {goneSinceRun} + {goneSinceRun === 1 ? 'model' : 'models'} this run built {goneSinceRun === 1 ? 'is' : 'are'} + no longer in the project — renamed or removed since, so + {goneSinceRun === 1 ? 'it is' : 'they are'} not drawn. +
+ {/if} +
+ (selection = s)} + showMinimap={false} + scrollZoom={false} + /> +
+ {@render sqlPane()} +
+{/if} diff --git a/frontend/src/lib/components/dbt/DbtRunResult.svelte b/frontend/src/lib/components/dbt/DbtRunResult.svelte new file mode 100644 index 0000000000..34d7886b30 --- /dev/null +++ b/frontend/src/lib/components/dbt/DbtRunResult.svelte @@ -0,0 +1,146 @@ + + +
+
+ {#each [{ k: 'success', label: 'passed', cls: 'text-green-600 dark:text-green-400' }, { k: 'warn', label: 'warned', cls: 'text-yellow-600 dark:text-yellow-400' }, { k: 'error', label: 'failed', cls: 'text-red-600 dark:text-red-400' }, { k: 'skipped', label: 'skipped', cls: 'text-secondary' }] as t (t.k)} + {@const n = (totals as Record)[t.k] ?? 0} + {#if n > 0} + {n} {t.label} + {/if} + {/each} + of {totals.total ?? nodes.length} nodes + + {run.command ?? 'build'} · {run.engine ?? ''} + {run.engine_version ?? ''} + +
+ + {#if nodes.length > 0} +
+ + + + + + + + + + + + {#each nodes as node (node.unique_id)} + {@const s = split(node.unique_id)} + {@const r = rank(node.status, node.outcome)} + + + + + + + + {/each} + +
NodeKindRelationRowsTime
+
+ + {#if r === 0} + + {:else if r === 1} + + {:else if r === 2} + + {:else} + + {/if} + + {s.name} + {#if node.message && r < 2} + {node.message} + {/if} +
+ {#if node.message && r < 2} +
+ {node.message} +
+ {/if} + 		+ + {#if s.kind === 'test'} + + {/if} + {s.kind} + + + {fmtRelation(node.relation_name) ?? ''} + + {node.rows_affected ?? ''} + + {fmtTime(node.execution_time)} +
+
+ {#if hasTests} +
+ A test's severity decides the outcome: dbt's own warn surfaces + without failing the job. +
+ {/if} + {/if} +
diff --git a/frontend/src/lib/components/dbt/parseDbtRun.test.ts b/frontend/src/lib/components/dbt/parseDbtRun.test.ts new file mode 100644 index 0000000000..47d6073ef7 --- /dev/null +++ b/frontend/src/lib/components/dbt/parseDbtRun.test.ts @@ -0,0 +1,193 @@ +import { describe, it, expect } from 'vitest' +import { + parseDbtRun, + relationOutcome, + splitRelation, + statusRank, + splitUniqueId, + nodeSelector +} from './parseDbtRun' + +const run = { + engine: 'dbt-core-1x', + engine_version: '1.12.0', + command: 'build', + totals: { total: 1, success: 1, error: 0, warn: 0, skipped: 0 }, + nodes: [{ unique_id: 'model.p.customers', status: 'success' }] +} + +describe('parseDbtRun', () => { + it('takes a successful run as-is', () => { + expect(parseDbtRun(run)?.engine).toBe('dbt-core-1x') + }) + + // The worker puts the same JSON in the error message after the exit-status + // line, and this is the case worth rendering: the failing node is what the + // user came for. + it('recovers the run from a failed job’s error message', () => { + const failed = { + error: { + name: 'ExecutionErr', + message: `execution error:\nNon-zero exit status for dbt build: 1\n\n${JSON.stringify(run)}` + } + } + expect(parseDbtRun(failed)?.totals?.total).toBe(1) + }) + + // The failures worth reading are the ones whose message carries braces of its + // own — a Jinja template, the compiled SQL, an adapter's own JSON — and the + // payload is appended after all of it. + it('finds the run past braces in the error text', () => { + const failed = { + error: { + name: 'ExecutionErr', + message: + 'execution error:\nCompilation Error in model x\n {{ ref("missing") }} depends on {"a": 1}\n' + + `Non-zero exit status for dbt build: 1\n\n${JSON.stringify(run)}` + } + } + expect(parseDbtRun(failed)?.totals?.total).toBe(1) + }) + + // `{nodes, totals}` alone is a shape an ordinary script can return, and it + // would then be rendered as somebody's dbt run. + it('does not claim an ordinary result that happens to have nodes and totals', () => { + expect(parseDbtRun({ nodes: [], totals: {} })).toBeUndefined() + expect(parseDbtRun({ engine: 'v8', nodes: [], totals: {} })).toBeUndefined() + }) + + it('accepts every engine the worker stamps', () => { + for (const engine of ['dbt-core-1x', 'dbt-core-2x', 'fusion']) { + expect(parseDbtRun({ ...run, engine })?.engine).toBe(engine) + } + }) + + // The payload carries one object per node, so a scan bounded by brace COUNT + // gives up on an ordinary project — a few hundred nodes, tests included — and + // silently loses the per-model table on exactly the runs it exists for. + it('finds the run in a payload with hundreds of nodes', () => { + const big = { + ...run, + totals: { total: 400, success: 399, error: 1, warn: 0, skipped: 0 }, + nodes: Array.from({ length: 400 }, (_, i) => ({ + unique_id: `model.p.m${i}`, + status: i === 0 ? 'error' : 'success' + })) + } + const failed = { + error: { + name: 'ExecutionErr', + message: + 'execution error:\nCompilation Error {{ ref("x") }} {"a": 1}\n\n' + + JSON.stringify(big, null, 2) + } + } + expect(parseDbtRun(failed)?.nodes?.length).toBe(400) + }) + + it('is undefined for anything unparseable', () => { + expect(parseDbtRun(undefined)).toBeUndefined() + expect(parseDbtRun('a string')).toBeUndefined() + expect(parseDbtRun({ error: { message: 'failed with {not json' } })).toBeUndefined() + }) +}) + +describe('statusRank', () => { + // dbt counts `partial success` in totals.error and a retry redoes it, so + // ranking it as a pass would contradict the job's own outcome. + it('ranks partial success with the failures', () => { + expect(statusRank('partial success')).toBe(statusRank('error')) + expect(statusRank('PARTIAL SUCCESS')).toBe(0) + }) + + // The worker publishes `unknown` for a status it does not recognise and counts + // it in `totals.error`; ranking it as a pass drew a green check on a node the + // same result called an error. + // The worker counts `no_op` in totals.skipped — dbt built nothing for that + // node — so a green check would claim a run that never happened. + it('ranks a no-op with the skips, not with the passes', () => { + expect(statusRank('no-op', 'no_op')).toBe(statusRank('skipped', 'skipped')) + expect(statusRank('success', 'no_op')).toBe(2) + }) + + it('ranks an unknown outcome with the failures, not with the passes', () => { + expect(statusRank('some-future-dbt-status', 'unknown')).toBe(0) + expect(statusRank('success', 'unknown')).toBe(0) + }) + + it('orders failed before warned before skipped before passed', () => { + expect( + ['success', 'skipped', 'warn', 'error'].sort((a, b) => statusRank(a) - statusRank(b)) + ).toEqual(['error', 'warn', 'skipped', 'success']) + }) +}) + +describe('splitUniqueId', () => { + it('splits kind from name and drops a generic test’s uniqueness hash', () => { + expect(splitUniqueId('model.jaffle.stg_orders')).toEqual({ + kind: 'model', + name: 'stg_orders' + }) + expect(splitUniqueId('test.jaffle.not_null_orders_id.4e687af8d0')).toEqual({ + kind: 'test', + name: 'not_null_orders_id' + }) + // A model whose name contains a dot keeps it: only tests carry the hash. + expect(splitUniqueId('model.jaffle.a.b').name).toBe('a.b') + }) +}) + +describe('relationOutcome', () => { + it('agrees with the worker classifier on every status it names', () => { + expect(relationOutcome('started')).toBe('running') + for (const s of ['success', 'pass', 'PASS', ' Success ']) { + expect(relationOutcome(s)).toBe('materialized') + } + // `partial success` built the relation and then failed its tests; the + // worker records it failed, so the colour must agree. + for (const s of ['error', 'fail', 'runtime error', 'partial success', 'PARTIAL SUCCESS']) { + expect(relationOutcome(s)).toBe('failed') + } + // Nothing was built, so nothing is coloured. + for (const s of ['warn', 'skipped', 'no-op', 'something new']) { + expect(relationOutcome(s)).toBeUndefined() + } + }) +}) + +describe('splitRelation', () => { + it('keeps a period that lives inside a quoted identifier', () => { + // The backend supports it, so rendering it as `v2.orders` names a + // relation that does not exist. + expect(splitRelation('"wh"."analytics.v2"."orders"')).toEqual(['wh', 'analytics.v2', 'orders']) + expect(splitRelation('"db"."schema"."name"')).toEqual(['db', 'schema', 'name']) + expect(splitRelation('db.schema.name')).toEqual(['db', 'schema', 'name']) + // BigQuery backticks and T-SQL brackets quote too. + expect(splitRelation('`proj`.`data.set`.`t`')).toEqual(['proj', 'data.set', 't']) + expect(splitRelation('[db].[my.schema].[t]')).toEqual(['db', 'my.schema', 't']) + }) + + // Every one of these dialects escapes its delimiter by doubling it. Dropping + // the pair renames the relation, and the manifest keeps the real spelling — + // so the run's status would be recorded against a key no graph node has. + it('keeps a delimiter the identifier escaped by doubling', () => { + expect(splitRelation('"wh"."schema"."a""b"')).toEqual(['wh', 'schema', 'a"b']) + expect(splitRelation('`proj`.`da``ta`.`t`')).toEqual(['proj', 'da`ta', 't']) + expect(splitRelation('[db].[my]]schema].[t]')).toEqual(['db', 'my]schema', 't']) + }) +}) + +describe('nodeSelector', () => { + // Verified against dbt-core 1.12, dbt-core 2.0.0-alpha.5 and fusion + // 2.0.0-preview.202: the intersection resolves to the one node whatever the + // project's `model-paths` is, while a path-derived FQN resolves to nothing + // as soon as that root is more than one segment deep. + it('intersects the name with its package, wherever the model sits', () => { + expect(nodeSelector('model.jaffle_shop.fct_orders')).toBe('fct_orders,package:jaffle_shop') + }) + + // Ambiguous across packages, but a selector dbt resolves rather than rejects. + it('falls back to the bare name without a package', () => { + expect(nodeSelector('fct_orders')).toBe('fct_orders') + }) +}) diff --git a/frontend/src/lib/components/dbt/parseDbtRun.ts b/frontend/src/lib/components/dbt/parseDbtRun.ts new file mode 100644 index 0000000000..aecd3dcf03 --- /dev/null +++ b/frontend/src/lib/components/dbt/parseDbtRun.ts @@ -0,0 +1,247 @@ +export type DbtNode = { + unique_id: string + status: string + /** Windmill's stable word for the same result, published beside dbt's own. + * Preferred wherever a decision is made: `status` is dbt's vocabulary and + * dbt may rename it. */ + outcome?: DbtOutcome + execution_time?: number + rows_affected?: number + relation_name?: string + message?: string +} + +export type DbtRun = { + engine?: string + engine_version?: string + command?: string + totals?: { total?: number; success?: number; error?: number; warn?: number; skipped?: number } + nodes?: DbtNode[] + /** The arguments the run actually used, as submitted. A `dbt retry` restores + * the failed run's arguments inside the worker, so the retry job's own args + * name only the run it resumed — this is the sole way to recover what it + * really ran with. */ + invocation_args?: Record +} + +/** The engines the worker stamps on a result. This is the discriminator: a + * `{nodes, totals}` shape alone is one an ordinary script can return, and it + * would then be rendered as somebody's dbt run. */ +const ENGINES = ['dbt-core-1x', 'dbt-core-2x', 'fusion'] + +function asDbtRun(v: unknown): DbtRun | undefined { + if (!v || typeof v !== 'object') return undefined + const o = v as Record + return ENGINES.includes(o.engine as string) && + Array.isArray(o.nodes) && + o.totals != undefined && + typeof o.totals === 'object' + ? (o as DbtRun) + : undefined +} + +/** + * The dbt invocation a job result describes, if it describes one. + * + * On success the result IS the run. On failure the worker puts the same JSON in + * the error message after the exit-status line, and that is the case worth + * rendering: the failing node is what the user came for. + */ +export function parseDbtRun(result: any): DbtRun | undefined { + const direct = asDbtRun(result) + if (direct) return direct + const msg = result?.error?.message + if (typeof msg !== 'string') return undefined + // The payload is appended pretty-printed, so its `{` is the only one at COLUMN + // ZERO — everything nested is indented, and dbt's own error text carries its + // braces mid-line. Counting braces instead needs a cap that a real project + // blows: forwards on an error full of them, backwards on one `{` per node. + for (const line of lineStarts(msg)) { + if (msg[line] !== '{') continue + try { + const run = asDbtRun(JSON.parse(msg.slice(line))) + if (run) return run + } catch { + // A `{` alone on a line inside the error text; the payload is later. + } + } + return undefined +} + +/** Index of the first character of each line, the payload's own `{` among them. */ +function* lineStarts(s: string): Generator { + let at = 0 + while (at !== -1 && at < s.length) { + yield at + const next = s.indexOf('\n', at) + at = next === -1 ? -1 : next + 1 + } +} + +/** Ordering rank of a node's status: 0 failed, 1 warned, 2 skipped, 3 passed. + * + * Both `unknown` and `no_op` rank where the RESULT counts them, not where the + * default would put them: the worker counts `unknown` in `totals.error` and + * `no_op` in `totals.skipped`, so falling through to 3 drew a green check on a + * node the same result called an error, and on one it never built. */ +export function statusRank(status: string, outcome?: DbtOutcome): number { + switch (outcome ?? classifyStatus(status)) { + case 'failed': + case 'unknown': + return 0 + case 'warned': + return 1 + case 'skipped': + case 'no_op': + return 2 + default: + return 3 + } +} + +/** The worker's stable vocabulary for a node result, published as `outcome`. */ +export type DbtOutcome = + | 'started' + | 'passed' + | 'failed' + | 'warned' + | 'skipped' + | 'no_op' + | 'unknown' + +/** + * dbt's node status, reduced to the outcomes the UI distinguishes. + * + * Only for results that predate `outcome`, or for the live event stream, which + * carries dbt's word alone. Anything holding a node from a job result should + * read `outcome` instead — that is the field the worker publishes precisely so + * this mapping is not the contract. + */ +function classifyStatus( + status: string +): 'started' | 'passed' | 'failed' | 'warned' | 'skipped' | 'other' { + // `partial success` is dbt's word for a node that built but whose tests + // failed. The worker counts it in `totals.error` and a retry redoes it, so + // showing it green would contradict the job's own outcome. + switch (status.trim().toLowerCase()) { + case 'started': + return 'started' + case 'success': + case 'pass': + return 'passed' + case 'error': + case 'fail': + case 'runtime error': + case 'partial success': + return 'failed' + case 'warn': + return 'warned' + case 'skipped': + return 'skipped' + default: + return 'other' + } +} + +/** + * The kind and name behind a dbt `unique_id`, which dbt builds as + * `..`. A generic test's name carries a trailing + * hash dbt adds for uniqueness; it is noise in a run summary. + */ +export function splitUniqueId(uniqueId: string): { kind: string; name: string } { + const parts = uniqueId.split('.') + const kind = parts[0] ?? '' + let name = parts.slice(2).join('.') + if (kind === 'test') name = name.replace(/\.[0-9a-f]{6,}$/, '') + return { kind, name: name || uniqueId } +} + +/** + * What a node's status says happened to the relation it builds, or `undefined` + * when it says nothing. + * + * Mirrors the worker's `classify_status`, and must keep mirroring it: the two + * decide the same thing about the same string, one for the record it writes and + * one for the colour drawn over it. `warn`, `skipped` and `no-op` leave the + * relation untouched, so they get no colour rather than a misleading one. + */ +export function relationOutcome( + status: string, + outcome?: DbtOutcome +): 'running' | 'materialized' | 'failed' | undefined { + switch (outcome ?? classifyStatus(status)) { + case 'started': + return 'running' + case 'passed': + return 'materialized' + case 'failed': + return 'failed' + // `warn`, `skipped` and `no-op` say nothing about the relation: nothing + // was written, so its state is whatever the last run left. + default: + return undefined + } +} + +/** + * dbt's `relation_name` split into its parts, honouring quoting. + * + * Mirrors the worker's `split_relation`: `"`, `` ` `` and `[` open a quoted + * identifier, and a `.` inside one is part of the name. Splitting on every `.` + * turns `"wh"."analytics.v2"."orders"` into a relation called `orders` in a + * schema called `v2` — a table that does not exist. + */ +export function splitRelation(relation: string): string[] { + const parts: string[] = [] + let current = '' + let quote: string | undefined + for (let i = 0; i < relation.length; i++) { + const c = relation[i] + if (quote !== undefined) { + const close = quote === '[' ? ']' : quote + if (c === close) { + // Doubled, which is how each of these dialects escapes its own + // delimiter: one literal character, not the end of the identifier. + if (relation[i + 1] === close) { + current += close + i++ + } else { + quote = undefined + } + } else current += c + } else if (c === '"' || c === '`' || c === '[') { + quote = c + } else if (c === '.') { + parts.push(current) + current = '' + } else { + current += c + } + } + parts.push(current) + return parts.map((p) => p.trim()) +} + +/** + * A selector naming exactly one node: `,package:`. + * + * The comma is dbt's intersection operator, so this reads "the node whose name + * is `` and whose package is ``" — one node, since dbt refuses + * two models of one name inside a package. A bare name would match the leaf of + * every package's FQN, and a package can ship a model whose name the project + * also uses. + * + * Not the FQN (`..`), which cannot be rebuilt from + * `original_file_path`: how many leading segments are the resource root is + * `model-paths`, and dropping exactly one turns `src/models/marts/orders.sql` + * into `pkg.models.marts.orders`, which dbt's matcher — equal lengths, from the + * front — resolves to nothing at all. + * + * Without a package, the bare name — ambiguous across packages, but a selector + * dbt resolves rather than one it rejects. + */ +export function nodeSelector(uniqueId: string): string { + const { name } = splitUniqueId(uniqueId) + const pkg = uniqueId.split('.')[1] + return pkg ? `${name},package:${pkg}` : name +} diff --git a/frontend/src/lib/components/dbt/previewRows.ts b/frontend/src/lib/components/dbt/previewRows.ts new file mode 100644 index 0000000000..48ff56ec4c --- /dev/null +++ b/frontend/src/lib/components/dbt/previewRows.ts @@ -0,0 +1,68 @@ +import { JobService } from '$lib/gen' + +/** A model's rows, as `dbt show` returns them. */ +export type DbtPreview = + | { pending: true } + | { rows: Record[]; node?: string; tookMs: number } + | { error: string } + +/** + * Preview one model's rows by running its own project's `dbt show`. + * + * A job, not a query: the rows come from the warehouse through the project's + * profile, with its vars and its adapter, which is the only place that knows how + * to resolve `ref()` and where the relation actually lives. `show` is therefore + * not a run-form command — it is what a table's preview is made of, here and on + * the run page's graph. + * + * `stillWanted` is asked before each poll and before the result is used, so a + * preview outlives neither the page that asked for it nor a navigation. + */ +export async function previewDbtRows(opts: { + workspace: string + scriptPath: string + /** Pins the preview to a deployed version, for a graph showing that version. */ + scriptHash?: string | number + /** One node: a model name, or `package.model` where two packages share one. */ + model: string + /** The run's own vars, so a descriptor with a required `{{ }}` var resolves. */ + vars?: Record + limit?: number + /** Extra top-level arguments — a run's `{{ placeholder }}` values. */ + args?: Record + stillWanted?: () => boolean +}): Promise { + const { workspace, scriptPath, scriptHash, model, vars, limit, args, stillWanted } = opts + const startedAt = Date.now() + const requestBody = { + ...(args ?? {}), + command: { label: 'show', vars: vars ?? {}, model, limit: limit ?? 25 } + } + try { + // By HASH whenever the caller pins one: the SQL on screen is that version's, + // and running the deployed one would show today's rows under it — or fail + // outright for a model since removed. + const id = scriptHash + ? await JobService.runScriptByHash({ + workspace, + hash: String(scriptHash), + requestBody + }) + : await JobService.runScriptByPath({ workspace, path: scriptPath, requestBody }) + // Polled rather than awaited: a preview is a job, and its engine may need + // provisioning on a cold worker. + for (let i = 0; i < 90; i++) { + await new Promise((r) => setTimeout(r, 1000)) + if (stillWanted && !stillWanted()) return undefined + const done = await JobService.getCompletedJobResultMaybe({ workspace, id }) + if (!done.completed) continue + const res = done.result as { node?: string; show?: Record[] } | undefined + return done.success && res?.show + ? { rows: res.show, node: res.node, tookMs: Date.now() - startedAt } + : { error: 'The preview job failed — open it from Runs for the detail.' } + } + return { error: 'The preview is still running; open it from Runs.' } + } catch (e) { + return { error: e instanceof Error ? e.message : String(e) } + } +} diff --git a/frontend/src/lib/components/flows/content/FlowInputs.svelte b/frontend/src/lib/components/flows/content/FlowInputs.svelte index 335a37b0c7..e275df3976 100644 --- a/frontend/src/lib/components/flows/content/FlowInputs.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputs.svelte @@ -12,7 +12,7 @@ import { Check, Code, Zap } from 'lucide-svelte' import SuspendDrawer from './SuspendDrawer.svelte' import { defaultScripts } from '$lib/stores' - import { defaultScriptLanguages, processLangs } from '$lib/scripts' + import { defaultScriptLanguages, processInlineLangs } from '$lib/scripts' import type { SupportedLanguage } from '$lib/common' import DefaultScripts from '$lib/components/DefaultScripts.svelte' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' @@ -48,7 +48,7 @@ let filter = $state('') let langs = $derived( - processLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) + processInlineLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) .map((l) => [defaultScriptLanguages[l], l]) .filter( (x) => $defaultScripts?.hidden == undefined || !$defaultScripts.hidden.includes(x[1]) diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index dc2cba93d4..7a55ca92fc 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -6,7 +6,7 @@ + + + + + + + + + + + diff --git a/frontend/src/lib/components/worker_group.ts b/frontend/src/lib/components/worker_group.ts index d0391b17c2..31d0cee7db 100644 --- a/frontend/src/lib/components/worker_group.ts +++ b/frontend/src/lib/components/worker_group.ts @@ -59,7 +59,8 @@ export const defaultTags = [ 'java', 'ruby', 'rlang', - 'duckdb' + 'duckdb', + 'dbt' // for related places search: ADD_NEW_LANG ] /** Strip cache_clear, null/undefined values, empty arrays and empty objects from a worker group config. */ diff --git a/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte b/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte new file mode 100644 index 0000000000..d472ce6d08 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte @@ -0,0 +1,173 @@ + + + + + + Where dbt projects in this workspace run. A project names a warehouse by name in its descriptor (profile.warehouse) and reaches + {DEFAULT_WAREHOUSE} when it names none, so a project carries no + connection of its own. The name is also what its tables are keyed on in the asset graph (dbt://{DEFAULT_WAREHOUSE}/schema/table), so two projects on one warehouse share their nodes. Each entry points at a resource, and + configuring one here is what makes it available: anyone who may run a dbt script builds with it + and reads its models, without being granted the resource, the same bargain workspace object + storage makes. + + + + + + Name + Resource + Target + + + + + {#each dbtSettings.warehouses as warehouse, i (i)} + + + + + + + + + + + + + + + + + + onDiscard?.()} + saveLabel="Save dbt warehouses" +/> diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 1e0a36af50..db32bf0274 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -42,7 +42,8 @@ import initRustParser, { parse_rust } from 'windmill-parser-wasm-rust' import initYamlParser, { parse_assets_ansible, parse_ansible, - parse_ansible_delegate + parse_ansible_delegate, + parse_dbt } from 'windmill-parser-wasm-yaml' import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp' import initNuParser, { parse_nu } from 'windmill-parser-wasm-nu' @@ -521,6 +522,9 @@ export async function inferArgs( } catch { inferedSchema = parseRSignatureFallback(code) } + } else if (language == 'dbt') { + await initWasmYaml() + inferedSchema = JSON.parse(parse_dbt(code)) // for related places search: ADD_NEW_LANG } else { return null diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 70dabb01f6..f67c0415f0 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -709,7 +709,7 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "language": { "type": "string", - "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative" + "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" }, "kind": { "type": "string", @@ -1218,7 +1218,7 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "language": { "type": "string", - "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative" + "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" }, "tag": { "type": "string" @@ -1250,7 +1250,7 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "language": { "type": "string", - "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative" + "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" }, "lock": { "type": "string", diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 91e310f5c9..178f4474e7 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -1345,6 +1345,43 @@ main <- function( return(toJSON(result, auto_unbox = TRUE)) } ` + +// A dbt script is a whole dbt project: the descriptor below is the script's +// content, and the project's own files live in its module bundle (the +// `