diff --git a/.agents/skills/update-sqlx/SKILL.md b/.agents/skills/update-sqlx/SKILL.md index ce4eaa5ff7..ccbd7baf3d 100644 --- a/.agents/skills/update-sqlx/SKILL.md +++ b/.agents/skills/update-sqlx/SKILL.md @@ -9,11 +9,75 @@ Windmill uses `SQLX_OFFLINE=true` in CI, which requires all `sqlx::query!` / `sq ## When to Run -Run after any change to SQL queries in Rust source files. Without it, CI will fail with: +Run after **adding or editing** a SQL query in Rust source. Without it, CI fails with: ``` error: `SQLX_OFFLINE=true` but there is no cached data for this query ``` +**Do NOT run it when a change only *removes* queries.** The cache is already complete for +CI; all that is left are orphaned entries, which are cosmetic and never break a build. +Running `prepare` to tidy them risks destroying the cache for no gain. Delete them +offline instead: for each `.sqlx/query-*.json`, normalize its `query` field (strip `\` +line-continuations, collapse whitespace) and check whether it still appears in any `.rs` +file. That detector reports ~48 false positives in a CE checkout — EE queries live in +`*_ee.rs` symlinks it cannot read — so **filter to the tables your change touched** and +delete only those. + +## Before You Run Anything + +1. **Back the cache up.** `prepare` deletes `.sqlx/` *before* regenerating, so any compile + failure leaves it gutted (observed: 2350 → 142 entries). + ```bash + cp -r backend/.sqlx /tmp/sqlx_backup # restore with: rm -rf backend/.sqlx && cp -r /tmp/sqlx_backup backend/.sqlx + ``` +2. **Point `DATABASE_URL` at THIS worktree's database.** `prepare` compiles every + `sqlx::query!` against the **live** database. Another worktree's DB lacks your + migrations, so every new-table query fails and takes the cache down with it. The + symptom is `relation "" does not exist` — that is a wrong + `DATABASE_URL`, not a broken query. See AGENTS.md → "Per-worktree ports and database". + +## Queries Inside Tests Need `--all-targets`, Which Fails In A CE Checkout + +`prepare` only caches queries in code it compiles, and `--workspace` alone does **not** +compile test targets. A `sqlx::query!` inside `tests/*.rs` therefore gets no entry, and CI +fails on the test target with the usual "no cached data" error even though the lib built +clean. `SQLX_OFFLINE=true cargo check --workspace --all-targets` is what reproduces it. + +Adding `--all-targets` caches them — and, in a CE checkout, **aborts partway through**: +`backend/tests/otel.rs` imports `windmill_common::otel_ee`, which exists only behind the +`private` feature, so the compile dies after `prepare` has already emptied `.sqlx/`. +Observed: 2435 → 4 entries, `error: cargo check failed with status: exit status: 101`. + +Do not fight it — the abort is a pre-existing EE gap, not something your change caused. +Take the entries you need and put the backup back: + +```bash +cd backend +cp -r .sqlx /tmp/sqlx_backup +ls /tmp/sqlx_backup | sort > /tmp/before.txt + +DATABASE_URL= \ + cargo sqlx prepare --workspace -- --workspace --features all_sqlx_features --all-targets +# expected to fail; it still wrote the entries it got to before dying + +ls .sqlx | sort > /tmp/after.txt +mkdir -p /tmp/newq +comm -13 /tmp/before.txt /tmp/after.txt | while read f; do cp ".sqlx/$f" /tmp/newq/; done + +rm -rf .sqlx && cp -r /tmp/sqlx_backup .sqlx && cp /tmp/newq/*.json .sqlx/ +``` + +**Read every file in `/tmp/newq` before copying it in** — print each one's `query` field and +confirm it is one of yours. The set is small (one per new test query), and anything else in +there means the run got further than you think. + +Then verify both targets, since the lib passing says nothing about the tests: + +```bash +SQLX_OFFLINE=true cargo check --workspace --features all_sqlx_features # lib +SQLX_OFFLINE=true cargo check -p --all-targets # tests +``` + ## The Problem `cargo sqlx prepare --workspace` **deletes all existing cache files** and regenerates only the ones found in the current compilation. If you don't compile with every feature flag (especially `private` for EE files), you will **silently delete EE query caches**, breaking CI for enterprise tests. @@ -68,7 +132,16 @@ But if it fails with EE compilation errors, use the safe procedure above. - **Never** run `cargo sqlx prepare --workspace` with only OSS features and commit the result — it will delete EE caches. - **Never** set `SQLX_OFFLINE=true` for local `cargo sqlx prepare` — use a live database per CLAUDE.md. (CI runs with `SQLX_OFFLINE=true`, which is why the cache must be complete.) +- **Never** run `prepare` without a `.sqlx` backup, or against a `DATABASE_URL` you have not confirmed belongs to this worktree. +- **Never** run `prepare` at all for a removal-only change. - **Never** skip the verification step (step 4 above). +- **Never** leave a `--all-targets` run's output in place after it aborts — it is a + near-empty cache. Restore the backup and graft on only the entries you verified. + +Step 4 compares against `origin/main` because step 1 restored from it, so the two agree. +If you did **not** run step 1 — auditing a branch's cache on its own, say — compare +against `git merge-base HEAD origin/main` instead: `origin/main` advances, so its newer +entries would read as losses on your branch. ## Verification diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..8ceb8e722a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Files a generator owns. Collapsed in review diffs and left out of language +# stats: reviewing them means reviewing the generator instead. +*.gen.ts linguist-generated=true diff --git a/.github/scripts/check-docs-links.mjs b/.github/scripts/check-docs-links.mjs index 122d098197..25647008b9 100644 --- a/.github/scripts/check-docs-links.mjs +++ b/.github/scripts/check-docs-links.mjs @@ -15,6 +15,16 @@ const CONCURRENCY = 24 const TIMEOUT_MS = 20000 const RETRIES = 2 +// Links whose target page is written but not yet deployed on windmill.dev: the app +// link is already the final slug, so a 404 is expected until the docs side ships. +// The value is why the entry exists, for whoever has to judge whether it still should. +const PENDING_DEPLOY = new Map([ + [ + 'https://www.windmill.dev/docs/getting_started/scripts_quickstart/dbt', + 'windmilldocs#1625 (dbt runtime quickstart)' + ] +]) + async function walk(dir) { const out = [] for (const entry of await readdir(dir, { withFileTypes: true })) { @@ -111,16 +121,45 @@ async function worker() { } await Promise.all(Array.from({ length: CONCURRENCY }, worker)) -const failures = results.filter((r) => !r.ok) -if (failures.length === 0) { - console.log(`\n✅ All ${allUrls.length} docs links are reachable.`) +// An entry claims one thing — the page is not published yet — and 404 is the only +// answer that means it. A timeout, 403 or 5xx on the same URL is a real fault, and +// suppressing it would also read as "still waiting" and defer the staleness check. +const isPendingDeploy = (r) => PENDING_DEPLOY.has(r.url) && r.status === 404 + +const pending = results.filter((r) => PENDING_DEPLOY.has(r.url)) +const waiting = results.filter(isPendingDeploy) +if (waiting.length) { + console.log(`\n⏳ ${waiting.length} link(s) waiting on a docs deploy:`) + for (const p of waiting.sort((a, b) => a.url.localeCompare(b.url))) { + console.log(` ${p.url}\n ${PENDING_DEPLOY.get(p.url)} — not live yet (${p.status})`) + } +} + +// An entry that outlived its reason exempts a URL from the check forever, so a stale +// one has to fail the job: a line in a green log is not read at release time. +const stale = [ + ...pending.filter((p) => p.ok).map((p) => [p.url, 'the page is live']), + ...[...PENDING_DEPLOY.keys()].filter((u) => !urls.has(u)).map((u) => [u, 'nothing references it']) +] + +const failures = results.filter((r) => !r.ok && !isPendingDeploy(r)) +if (failures.length === 0 && stale.length === 0) { + console.log(`\n✅ No broken docs links (${allUrls.length} checked).`) process.exit(0) } -console.log(`\n❌ ${failures.length} broken docs link(s):`) -for (const f of failures.sort((a, b) => a.url.localeCompare(b.url))) { - console.log(`\n ${f.url}`) - console.log(` status: ${f.error ? `error (${f.error})` : f.status}`) - for (const file of urls.get(f.url)) console.log(` ↳ ${file}`) +if (failures.length) { + console.log(`\n❌ ${failures.length} broken docs link(s):`) + for (const f of failures.sort((a, b) => a.url.localeCompare(b.url))) { + console.log(`\n ${f.url}`) + console.log(` status: ${f.error ? `error (${f.error})` : f.status}`) + for (const file of urls.get(f.url)) console.log(` ↳ ${file}`) + } +} +if (stale.length) { + console.log(`\n❌ ${stale.length} PENDING_DEPLOY entr(ies) to delete from this script:`) + for (const [url, why] of stale.sort((a, b) => a[0].localeCompare(b[0]))) { + console.log(`\n ${url}\n ${why}`) + } } process.exit(1) diff --git a/AGENTS.md b/AGENTS.md index 90c0060ae2..5e0c5885c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,9 @@ Open-source platform for internal tools, workflows, API integrations, background - **Validation**: `docs/validation.md` — what checks to run based on what you changed - **Unreleased SDK changes**: `docs/wac-sdk-e2e.md` — exercising a client change on a real worker +- **Agent workers**: `docs/agent-worker-e2e.md` — building and running one locally. An agent + reaches the DB only through the API, so `Connection::Http` paths are never taken by a plain + `cargo run`; a normal build cannot start one at all. - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. @@ -21,9 +24,15 @@ Open-source platform for internal tools, workflows, API integrations, background - **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. +- **Session recorder**: `frontend/src/lib/components/recording/` is also the recorder `wmill app dev --recording` serves, vendored into the CLI as `cli/src/commands/app/devRecorderBundle.gen.ts`. After changing `rawAppSnapshot.ts` or `rawAppRecording.svelte.ts`, run `bun run gen:dev-recorder` from `cli/` (`cli/test/dev_recorder_bundle_unit.test.ts` fails otherwise). ## Dev Environment +> **In a git worktree, the ports and database below are NOT the ones to use.** Each +> worktree gets its own backend port, frontend port and Postgres database, so the +> defaults in this section apply only to a plain single checkout. **Discover the real +> values before running anything** — see "Per-worktree ports and database" below. + - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) - **DuckDB local jobs**: before running DuckDB scripts locally, build the FFI shared library with `cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. Re-run it after clean builds or when `backend/target/debug/libwindmill_duckdb_ffi_internal.*` is missing. The bundled DuckDB compile (~2min) is cached in a per-user dir shared across worktrees, so a fresh worktree reuses it and the build is near-instant. - **Data pipelines (DuckLake) from source**: a plain `cargo run` (even `--features quickjs`) advertises a `duckdb` worker tag but **cannot** execute DuckDB scripts and has **no** working S3 proxy (DuckLake writes 404). Build CE DuckLake with `cargo run --features quickjs,duckdb,parquet,private` (add `,python` for Python scripts, `,enterprise,license` for EE) **and** build the FFI (bullet above). See `backend/CLAUDE.md` → "Running data pipelines (DuckLake) from source" for the exact feature sets and the two feature-gate gotchas. @@ -33,6 +42,37 @@ Open-source platform for internal tools, workflows, API integrations, background - **Instance settings**: navigate to `/#superadmin-settings` - **Migrations**: use `cargo sqlx migrate add -r ` from `backend/` to create new migrations (never generate timestamps manually) +### Per-worktree ports and database + +A worktree's `.env` / `.env.local` (repo root) and `backend/.env` hold its own +`DATABASE_URL` and `PORT`; the database is typically `windmill_` +(branch `dbt-runtime` → `windmill_dbt_runtime`). Read them, or discover from what is +already running: + +```bash +psql postgres://postgres:changeme@localhost:5432/postgres -tAc \ + "select datname from pg_database where datname like 'windmill%'" | grep "$(git branch --show-current | tr - _)" +# the port the frontend actually proxies to (REMOTE of this worktree's vite): +for p in $(pgrep -f vite); do case "$(readlink /proc/$p/cwd)" in *"$(basename "$(git rev-parse --show-toplevel)")"*) + tr '\0' '\n' < /proc/$p/environ | grep -E '^REMOTE=|^PORT=';; esac; done +``` + +Getting these wrong is not a cheap mistake: + +- **`DATABASE_URL` pointed at another worktree's database silently destroys the sqlx + cache.** `cargo run` and `cargo sqlx prepare` both compile `sqlx::query!` against the + **live** database, so the wrong one fails with `relation "" does not + exist` — and `prepare` deletes the whole `.sqlx/` directory *before* it fails, leaving + it gutted. Always `cp -r backend/.sqlx /sqlx_backup` first (see the `update-sqlx` + skill). +- **The frontend proxies to its own worktree's backend port, not 8000.** Starting a + backend on the wrong port leaves the UI up but every API call 502s, which reads like an + application bug rather than a misconfiguration. +- **Kill backends by pid scoped to this worktree's cwd** (`readlink /proc//cwd`), + never `pkill -f target/debug/windmill` — that kills every sibling worktree's backend. + Beware that a `pgrep -f ""` in a shell whose own command line contains + `` matches the shell itself. + ## Verifying Frontend Changes After modifying frontend code, drive the running dev server with the **Playwright MCP** to verify the change in a real browser — don't claim a UI change works without exercising it. @@ -55,6 +95,25 @@ Typical flow: If you cannot exercise a UI change (no dev server, etc.), say so explicitly rather than claiming success. +## Verifying Backend Changes + +`cargo check` and the unit tests do not exercise a worker code path. **If you changed how +a job runs — an executor, `handle_child`, anything spawning or reading from a +subprocess — run an actual job of that kind** and confirm it completed, then say so. +Whole classes of defect compile and unit-test clean: + +- **Stack overflow from a large buffer in an async block.** An array declared across an + `.await` is baked into the future's state; once that future is boxed a few layers deep + by the job poller, two 16 KB arrays abort the worker *process* (`thread + 'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers + (`vec![0u8; N]`, not `[0u8; N]`). +- Deadlocks from draining only one of a child's pipes, missed cancellation or timeout + propagation, and anything depending on the real engine's output format. + +A crash like this takes down every job on that worker, not just yours, so check the +backend log after the run rather than only the job's own status. If you cannot run one, +say which path went unexercised instead of implying it was verified. + ## Banned Patterns ### `$bindable(default_value)` on optional props diff --git a/CHANGELOG.md b/CHANGELOG.md index a69d7b0389..0ac92305b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,63 @@ # Changelog +## [1.777.1](https://github.com/windmill-labs/windmill/compare/v1.777.0...v1.777.1) (2026-08-03) + + +### Bug Fixes + +* return result.json and stdout results from sandboxed containers ([#10460](https://github.com/windmill-labs/windmill/issues/10460)) ([d509551](https://github.com/windmill-labs/windmill/commit/d5095515ed007d4e7fbfc1a15582c550d3293ece)) + +## [1.777.0](https://github.com/windmill-labs/windmill/compare/v1.776.0...v1.777.0) (2026-08-03) + + +### Features + +* add session recording to wmill app dev ([#10457](https://github.com/windmill-labs/windmill/issues/10457)) ([2105540](https://github.com/windmill-labs/windmill/commit/2105540cca7cc7bdced9e06ef3ad1ed54732feef)) +* give dbt its own editor with an explicitly refreshed model graph ([#10448](https://github.com/windmill-labs/windmill/issues/10448)) ([baefa13](https://github.com/windmill-labs/windmill/commit/baefa1345b7b7a4110257a53a417d4fc229d0149)) +* show an on-behalf-of badge on the script and flow detail pages ([#10452](https://github.com/windmill-labs/windmill/issues/10452)) ([9bafb7e](https://github.com/windmill-labs/windmill/commit/9bafb7ebd2b8571f1adf5a86b681af8d07660a50)) + + +### Bug Fixes + +* keep uri and method on request logs under RUST_LOG=error ([#10462](https://github.com/windmill-labs/windmill/issues/10462)) ([0466ea2](https://github.com/windmill-labs/windmill/commit/0466ea201952bd382d893bd973eedaa7183b597e)) +* report a missing worker tag instead of spinning in data table UIs ([#10456](https://github.com/windmill-labs/windmill/issues/10456)) ([eca24bd](https://github.com/windmill-labs/windmill/commit/eca24bdfb5c7a49c428d541ca06407d1635cc3b6)) + +## [1.776.0](https://github.com/windmill-labs/windmill/compare/v1.775.2...v1.776.0) (2026-08-01) + + +### Features + +* **frontend:** add missing resource type icons and show them in the resource picker ([#10407](https://github.com/windmill-labs/windmill/issues/10407)) ([705c90d](https://github.com/windmill-labs/windmill/commit/705c90debda87451fcdc32f15f086080ab7dc5f9)) +* **git-sync:** dedicated base url for GitHub webhook delivery ([#10411](https://github.com/windmill-labs/windmill/issues/10411)) ([318c9f0](https://github.com/windmill-labs/windmill/commit/318c9f00739bfd43bd5d6afed4ae12f3f1a565d1)) +* let the merge UI target an arbitrary workspace ([#10417](https://github.com/windmill-labs/windmill/issues/10417)) ([61f2d8d](https://github.com/windmill-labs/windmill/commit/61f2d8dc6ab980c1aba959b853d505f09299ed9c)) +* make job subprocess oom_score_adj configurable ([#10443](https://github.com/windmill-labs/windmill/issues/10443)) ([2508417](https://github.com/windmill-labs/windmill/commit/25084170d7449dfa06dd758bc3783580916cfa95)) +* make the fork lineage the only deploy relationship ([#10410](https://github.com/windmill-labs/windmill/issues/10410)) ([81b23a2](https://github.com/windmill-labs/windmill/commit/81b23a2ba0ee1001241e4fe6066c07526ab640f6)) +* run dbt projects as a first-class Windmill runtime ([#10326](https://github.com/windmill-labs/windmill/issues/10326)) ([032300e](https://github.com/windmill-labs/windmill/commit/032300e28eba9f8e790f16e894bb00fff22eb296)) +* stamp webhook trigger_kind on token-driven job runs ([#10431](https://github.com/windmill-labs/windmill/issues/10431)) ([dda5976](https://github.com/windmill-labs/windmill/commit/dda59767c2b997e675ffe799bc6b1dcc79c2a52d)) +* sync data table migrations to git, gated by a new object type ([#10436](https://github.com/windmill-labs/windmill/issues/10436)) ([bfc3f52](https://github.com/windmill-labs/windmill/commit/bfc3f5242a0d9a833d12859f96ea8f1aa67d362b)) + + +### Bug Fixes + +* add apps:run to the token scope picker and confine path-scoped app tokens ([#10428](https://github.com/windmill-labs/windmill/issues/10428)) ([c69f080](https://github.com/windmill-labs/windmill/commit/c69f08073a657ee6d91bfdbd19e8639a392fd77a)) +* **ai:** pass only the output of a nested agent tool to the parent ([#10416](https://github.com/windmill-labs/windmill/issues/10416)) ([7d097d2](https://github.com/windmill-labs/windmill/commit/7d097d25c3bba89d708c21a2fc3a1a8e0099a5c5)) +* **ai:** route Azure OpenAI agent steps through the Responses API ([#10404](https://github.com/windmill-labs/windmill/issues/10404)) ([94bcc00](https://github.com/windmill-labs/windmill/commit/94bcc00554423eb9e4056dd33ceff1b07263e9ec)) +* app progress bar stuck on running, and misreporting queued/canceled jobs as errors ([#10409](https://github.com/windmill-labs/windmill/issues/10409)) ([5579913](https://github.com/windmill-labs/windmill/commit/557991360a5c920909cac45e335c11b35bd2d880)) +* apply default workspace dependencies to raw app runnables ([#10427](https://github.com/windmill-labs/windmill/issues/10427)) ([38b6099](https://github.com/windmill-labs/windmill/commit/38b6099b4c5039232cf306a01ee814b3ec5dc04c)) +* carry the token label into job-run audit rows ([#10433](https://github.com/windmill-labs/windmill/issues/10433)) ([02c4a9e](https://github.com/windmill-labs/windmill/commit/02c4a9e515b3ef9c0e759211a6b7cbd874a778b0)) +* **cli:** lint against the checkout's schema, not the published validator ([#10418](https://github.com/windmill-labs/windmill/issues/10418)) ([a372ae0](https://github.com/windmill-labs/windmill/commit/a372ae0c04d0849932d83a76d38b5a9e507077af)) +* credit the token owner instead of the token label in the audit trail ([#10423](https://github.com/windmill-labs/windmill/issues/10423)) ([3716a71](https://github.com/windmill-labs/windmill/commit/3716a71fd76f66b58bc29977b4f6a10ed97cea16)) +* **flows:** mint fresh orchestration token so long steps don't expire the result-fetch JWT ([#10415](https://github.com/windmill-labs/windmill/issues/10415)) ([2e249ff](https://github.com/windmill-labs/windmill/commit/2e249ff8922c152f410cd40b88514f5dad875b85)) +* **forks:** record fork changes that never reached the diff tally ([#10403](https://github.com/windmill-labs/windmill/issues/10403)) ([f9a547b](https://github.com/windmill-labs/windmill/commit/f9a547b8b8e4982346607acae4e8e7646529c374)) +* harden flow-orchestration token refresh (mint from job_perms) ([#10419](https://github.com/windmill-labs/windmill/issues/10419)) ([e0d6dc1](https://github.com/windmill-labs/windmill/commit/e0d6dc1a1997514bfcaa8615de61daff4a2f81ca)) +* honor on-behalf-of when a workflow step dispatches a script or flow ([#10437](https://github.com/windmill-labs/windmill/issues/10437)) ([2b525d2](https://github.com/windmill-labs/windmill/commit/2b525d28dbcda4a4b0f626ad40b0c4d79cfbcd53)) +* keep native triggers attached when a runnable is renamed ([#10432](https://github.com/windmill-labs/windmill/issues/10432)) ([bd71566](https://github.com/windmill-labs/windmill/commit/bd7156682d4d4b78e01fa316580e632152cc5b62)) +* make /usr/bin/coursier self-contained so java jobs work air-gapped ([#10414](https://github.com/windmill-labs/windmill/issues/10414)) ([7e64960](https://github.com/windmill-labs/windmill/commit/7e649604db2047bf8587b46628aa74051e1b9409)) +* make on_behalf_of control permissions for scripts and flows ([#10438](https://github.com/windmill-labs/windmill/issues/10438)) ([fb82748](https://github.com/windmill-labs/windmill/commit/fb82748296cd0f81ad8d21c30c12e172a6477173)) +* pre-warm coursier bootstrap cache at the worker's cache path ([#10413](https://github.com/windmill-labs/windmill/issues/10413)) ([ed9dfc5](https://github.com/windmill-labs/windmill/commit/ed9dfc5de684197e14f3c4a3a6873c0e1a55229a)) +* run the init script before dedicated workers install dependencies ([#10412](https://github.com/windmill-labs/windmill/issues/10412)) ([43a684d](https://github.com/windmill-labs/windmill/commit/43a684d7432837fcf00a97712f28d7c474a58402)) +* sidebar workspace toggle navigates home when already in workspace mode ([#10405](https://github.com/windmill-labs/windmill/issues/10405)) ([9cd6a70](https://github.com/windmill-labs/windmill/commit/9cd6a70f6ace956f679c1ce1fb46543cda569183)) + ## [1.775.2](https://github.com/windmill-labs/windmill/compare/v1.775.1...v1.775.2) (2026-07-29) diff --git a/backend/.sqlx/query-00a7fccde8bc2075642ba02f4015d752ba7d67966ed03c0bf3414c842c049b3a.json b/backend/.sqlx/query-00a7fccde8bc2075642ba02f4015d752ba7d67966ed03c0bf3414c842c049b3a.json new file mode 100644 index 0000000000..2d3292c34b --- /dev/null +++ b/backend/.sqlx/query-00a7fccde8bc2075642ba02f4015d752ba7d67966ed03c0bf3414c842c049b3a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, on_behalf_of, on_behalf_of_email)\n VALUES ('test-workspace', 'u/test-user/obo_flow', '', '', $1, 'test-user', NOW(), 'u/test-user-2', 'test2@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "00a7fccde8bc2075642ba02f4015d752ba7d67966ed03c0bf3414c842c049b3a" +} diff --git a/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json b/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json deleted file mode 100644 index 13536a244e..0000000000 --- a/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[])", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e" -} diff --git a/backend/.sqlx/query-015b63b1fbdf95fc76138fcf0aed03ac8948cfcf071c7f5df574c5f7003545cd.json b/backend/.sqlx/query-015b63b1fbdf95fc76138fcf0aed03ac8948cfcf071c7f5df574c5f7003545cd.json new file mode 100644 index 0000000000..9d3b9e9287 --- /dev/null +++ b/backend/.sqlx/query-015b63b1fbdf95fc76138fcf0aed03ac8948cfcf071c7f5df574c5f7003545cd.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM flow WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "015b63b1fbdf95fc76138fcf0aed03ac8948cfcf071c7f5df574c5f7003545cd" +} diff --git a/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json b/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json index a0d8b66a26..6c282c7db8 100644 --- a/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json +++ b/backend/.sqlx/query-01732ca02b1888145c48c4e51e5b5829657224a743af9c0b2d5a140ad70e13dd.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-0da2425ff8ee737104cd9f2650f1ffba1511715ac1f9270939d79fb82f8e88d0.json b/backend/.sqlx/query-024fa8a99a7967a56cd5ac9486ef728f2de0800eeb877fd29aca2fba3029aa55.json similarity index 54% rename from backend/.sqlx/query-0da2425ff8ee737104cd9f2650f1ffba1511715ac1f9270939d79fb82f8e88d0.json rename to backend/.sqlx/query-024fa8a99a7967a56cd5ac9486ef728f2de0800eeb877fd29aca2fba3029aa55.json index 36e9ac3e04..d44cd42986 100644 --- a/backend/.sqlx/query-0da2425ff8ee737104cd9f2650f1ffba1511715ac1f9270939d79fb82f8e88d0.json +++ b/backend/.sqlx/query-024fa8a99a7967a56cd5ac9486ef728f2de0800eeb877fd29aca2fba3029aa55.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE azure_trigger SET email = $1 WHERE email = $2", + "query": "UPDATE flow SET on_behalf_of = $1 WHERE on_behalf_of = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "0da2425ff8ee737104cd9f2650f1ffba1511715ac1f9270939d79fb82f8e88d0" + "hash": "024fa8a99a7967a56cd5ac9486ef728f2de0800eeb877fd29aca2fba3029aa55" } diff --git a/backend/.sqlx/query-03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5.json b/backend/.sqlx/query-03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5.json new file mode 100644 index 0000000000..f99e53856c --- /dev/null +++ b/backend/.sqlx/query-03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest)\n VALUES ($1, $2, $3, $4, 'd2')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "03238f44f8d4db3ed406f853b5afd945796a2657e58cf1c5086d971e652803d5" +} diff --git a/backend/.sqlx/query-04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480.json b/backend/.sqlx/query-04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480.json new file mode 100644 index 0000000000..4a632d868e --- /dev/null +++ b/backend/.sqlx/query-04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, runnable_path FROM v2_job\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "04896f435eee78f5ac604f5a57c8d35a313fc321df788e1c701e11f60da46480" +} diff --git a/backend/.sqlx/query-058df18b148867dbb8bcf9e10c485d276b5af4b2b972fb335e41077a029c3196.json b/backend/.sqlx/query-058df18b148867dbb8bcf9e10c485d276b5af4b2b972fb335e41077a029c3196.json new file mode 100644 index 0000000000..1dd8e03e43 --- /dev/null +++ b/backend/.sqlx/query-058df18b148867dbb8bcf9e10c485d276b5af4b2b972fb335e41077a029c3196.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "058df18b148867dbb8bcf9e10c485d276b5af4b2b972fb335e41077a029c3196" +} diff --git a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json b/backend/.sqlx/query-09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e.json similarity index 92% rename from backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json rename to backend/.sqlx/query-09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e.json index 0eb26d5dc0..8840ce4958 100644 --- a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json +++ b/backend/.sqlx/query-09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as,\n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "query": "SELECT\n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as,\n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n WHERE j.id = $1 AND j.workspace_id = $2", "describe": { "columns": [ { @@ -119,7 +119,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } @@ -137,7 +138,7 @@ }, { "ordinal": 14, - "name": "trigger_kind: JobTriggerKind", + "name": "trigger_kind: TriggerKindLabel", "type_info": { "Custom": { "name": "job_trigger_kind", @@ -236,5 +237,5 @@ true ] }, - "hash": "67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8" + "hash": "09ae123099f05fe3db8bd67ca2c0d778a27619dd7fad71ba656f6b54c4915d1e" } diff --git a/backend/.sqlx/query-0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6.json b/backend/.sqlx/query-0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6.json new file mode 100644 index 0000000000..99bf8ee009 --- /dev/null +++ b/backend/.sqlx/query-0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_edge WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0b3754e11677390837ec7ab00ba5443dc634b8628e90cce879952c8c4319a6a6" +} diff --git a/backend/.sqlx/query-0c416eee574142b7a709a6e3e9a47d436f407781ff2bee820d3234317f46f2a6.json b/backend/.sqlx/query-0c416eee574142b7a709a6e3e9a47d436f407781ff2bee820d3234317f46f2a6.json new file mode 100644 index 0000000000..6f1603894b --- /dev/null +++ b/backend/.sqlx/query-0c416eee574142b7a709a6e3e9a47d436f407781ff2bee820d3234317f46f2a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of'], to_jsonb('u/' || $1))) WHERE value->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0c416eee574142b7a709a6e3e9a47d436f407781ff2bee820d3234317f46f2a6" +} diff --git a/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json b/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json new file mode 100644 index 0000000000..e74bd5de0e --- /dev/null +++ b/backend/.sqlx/query-0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0c5b02b6b70fb8fd2ab3e6c57897038750a44a67360d342b6ef705ef2e4d3007" +} diff --git a/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json b/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json index 238522a3ff..a9e75189af 100644 --- a/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json +++ b/backend/.sqlx/query-0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } @@ -55,7 +56,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json b/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json index 4194e7e9cc..0bc0c3ce03 100644 --- a/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json +++ b/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json @@ -35,7 +35,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-117b6de99cd75e279f738226d9455c506ac1d40163b4f3c50279a93e4cded5fa.json b/backend/.sqlx/query-117b6de99cd75e279f738226d9455c506ac1d40163b4f3c50279a93e4cded5fa.json new file mode 100644 index 0000000000..c7df8b7610 --- /dev/null +++ b/backend/.sqlx/query-117b6de99cd75e279f738226d9455c506ac1d40163b4f3c50279a93e4cded5fa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of_email'], to_jsonb($1::text))) WHERE typ IN ('script', 'flow') AND value->>'on_behalf_of_email' = $2 AND (value->>'on_behalf_of' IS NULL OR value->>'on_behalf_of' NOT LIKE 'g/%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "117b6de99cd75e279f738226d9455c506ac1d40163b4f3c50279a93e4cded5fa" +} diff --git a/backend/.sqlx/query-11813108dbf6b104eba968c3609c74ac5a589542d3e76b1e756a82fb19d49ee8.json b/backend/.sqlx/query-11813108dbf6b104eba968c3609c74ac5a589542d3e76b1e756a82fb19d49ee8.json index cef272c219..b8e3bb8927 100644 --- a/backend/.sqlx/query-11813108dbf6b104eba968c3609c74ac5a589542d3e76b1e756a82fb19d49ee8.json +++ b/backend/.sqlx/query-11813108dbf6b104eba968c3609c74ac5a589542d3e76b1e756a82fb19d49ee8.json @@ -34,7 +34,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-1253d0f62254b30979a1c8b41d5dc1a57a0253f9a76721e7b8965905613f2ace.json b/backend/.sqlx/query-1253d0f62254b30979a1c8b41d5dc1a57a0253f9a76721e7b8965905613f2ace.json new file mode 100644 index 0000000000..21dbc92b79 --- /dev/null +++ b/backend/.sqlx/query-1253d0f62254b30979a1c8b41d5dc1a57a0253f9a76721e7b8965905613f2ace.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of, on_behalf_of_email)\n VALUES ('test-workspace', 'f/shared/obo', 1099, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2', 'test2@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1253d0f62254b30979a1c8b41d5dc1a57a0253f9a76721e7b8965905613f2ace" +} diff --git a/backend/.sqlx/query-12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66.json b/backend/.sqlx/query-12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66.json new file mode 100644 index 0000000000..a797527421 --- /dev/null +++ b/backend/.sqlx/query-12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest)\n VALUES ($1, $2, $3, $4, 'd')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "12912ebe5df4eda15976e7c3a2f0501ed3098f33a234f283def6932ce491eb66" +} diff --git a/backend/.sqlx/query-13ab0cda87d04b9c1fb52b46e0211d02b24375643a030efdbfab331cdc248349.json b/backend/.sqlx/query-13ab0cda87d04b9c1fb52b46e0211d02b24375643a030efdbfab331cdc248349.json new file mode 100644 index 0000000000..66432001da --- /dev/null +++ b/backend/.sqlx/query-13ab0cda87d04b9c1fb52b46e0211d02b24375643a030efdbfab331cdc248349.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin)\n VALUES (encode(sha256('LONG_TOKEN'::bytea), 'hex'), 'LONG_TOKEN', 'LONG_TOKEN', $1, 'long', true)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "13ab0cda87d04b9c1fb52b46e0211d02b24375643a030efdbfab331cdc248349" +} diff --git a/backend/.sqlx/query-14a826d794da720981368c77a4f9dac27307833fafaa0ea232819aecd4d62773.json b/backend/.sqlx/query-14a826d794da720981368c77a4f9dac27307833fafaa0ea232819aecd4d62773.json new file mode 100644 index 0000000000..6604c44c97 --- /dev/null +++ b/backend/.sqlx/query-14a826d794da720981368c77a4f9dac27307833fafaa0ea232819aecd4d62773.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE native_trigger\n SET webhook_token_hash = $1, service_config = $2, error = NULL, updated_at = NOW()\n WHERE\n workspace_id = $3\n AND service_name = $4\n AND external_id = $5\n AND updated_at = $6\n RETURNING 1 AS \"applied!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "applied!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text", + "Timestamptz" + ] + }, + "nullable": [ + null + ] + }, + "hash": "14a826d794da720981368c77a4f9dac27307833fafaa0ea232819aecd4d62773" +} diff --git a/backend/.sqlx/query-164d94369014ce77642984818a0e436a87a1bd1c56fcaafbfae2ee45f62fe4e1.json b/backend/.sqlx/query-164d94369014ce77642984818a0e436a87a1bd1c56fcaafbfae2ee45f62fe4e1.json new file mode 100644 index 0000000000..875e8b823c --- /dev/null +++ b/backend/.sqlx/query-164d94369014ce77642984818a0e436a87a1bd1c56fcaafbfae2ee45f62fe4e1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of_email'], to_jsonb($1::text)) WHERE policy->>'on_behalf_of_email' = $2 AND (policy->>'on_behalf_of' IS NULL OR policy->>'on_behalf_of' NOT LIKE 'g/%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "164d94369014ce77642984818a0e436a87a1bd1c56fcaafbfae2ee45f62fe4e1" +} diff --git a/backend/.sqlx/query-16e8d3f75ce4b5b18cefc25ebb670b506677681394606426964491488d64c62c.json b/backend/.sqlx/query-16e8d3f75ce4b5b18cefc25ebb670b506677681394606426964491488d64c62c.json deleted file mode 100644 index 06a3ee417e..0000000000 --- a/backend/.sqlx/query-16e8d3f75ce4b5b18cefc25ebb670b506677681394606426964491488d64c62c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO azure_trigger (\n azure_resource_path, azure_mode, scope_resource_id, topic_name,\n subscription_name, event_type_filters, push_auth_config, path, script_path,\n is_flow, workspace_id, edited_by, email, edited_at, extra_perms, server_id,\n last_server_ping, error, mode, permissioned_as, error_handler_path,\n error_handler_args, retry, labels\n )\n SELECT\n azure_resource_path, azure_mode, scope_resource_id, topic_name,\n subscription_name, event_type_filters, push_auth_config, path, script_path,\n is_flow, $1, edited_by, email, edited_at, extra_perms, NULL,\n NULL, NULL, 'disabled'::TRIGGER_MODE, permissioned_as, error_handler_path,\n error_handler_args, retry, labels\n FROM azure_trigger WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "16e8d3f75ce4b5b18cefc25ebb670b506677681394606426964491488d64c62c" -} diff --git a/backend/.sqlx/query-175a732180336a801d1c2f41854269a2ce2160349f163b714cee77b5ed4b92f5.json b/backend/.sqlx/query-175a732180336a801d1c2f41854269a2ce2160349f163b714cee77b5ed4b92f5.json new file mode 100644 index 0000000000..a25e9afac0 --- /dev/null +++ b/backend/.sqlx/query-175a732180336a801d1c2f41854269a2ce2160349f163b714cee77b5ed4b92f5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "175a732180336a801d1c2f41854269a2ce2160349f163b714cee77b5ed4b92f5" +} diff --git a/backend/.sqlx/query-19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da.json b/backend/.sqlx/query-19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da.json new file mode 100644 index 0000000000..76404ffce5 --- /dev/null +++ b/backend/.sqlx/query-19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "19d19f3ed995ac68d9692c8fdf96836099dd325fa25d3660eb766a895b07c1da" +} diff --git a/backend/.sqlx/query-1a3fb9f51abd42f1ac4c7597f985a687bed32d2905a446c4e4b33f0b853e8e54.json b/backend/.sqlx/query-1a3fb9f51abd42f1ac4c7597f985a687bed32d2905a446c4e4b33f0b853e8e54.json new file mode 100644 index 0000000000..1c37708a50 --- /dev/null +++ b/backend/.sqlx/query-1a3fb9f51abd42f1ac4c7597f985a687bed32d2905a446c4e4b33f0b853e8e54.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.script_path = $2 AND g.script_hash IS NULL\n AND g.permissioned_as IS NOT DISTINCT FROM $4\n AND g.job_id NOT IN (\n SELECT job_id FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL\n AND permissioned_as IS NOT DISTINCT FROM $4\n ORDER BY ingested_at DESC LIMIT $3)\n RETURNING g.job_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1a3fb9f51abd42f1ac4c7597f985a687bed32d2905a446c4e4b33f0b853e8e54" +} diff --git a/backend/.sqlx/query-1b18f9fdcc0fbc3d5a1328776a804e2943193ad7d28b67df4e80baf38fc19271.json b/backend/.sqlx/query-1b18f9fdcc0fbc3d5a1328776a804e2943193ad7d28b67df4e80baf38fc19271.json new file mode 100644 index 0000000000..2dd66d2df6 --- /dev/null +++ b/backend/.sqlx/query-1b18f9fdcc0fbc3d5a1328776a804e2943193ad7d28b67df4e80baf38fc19271.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, on_behalf_of_email AS email FROM script WHERE workspace_id = 'test-workspace' AND path LIKE 'u/test-user/s%' ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "1b18f9fdcc0fbc3d5a1328776a804e2943193ad7d28b67df4e80baf38fc19271" +} diff --git a/backend/.sqlx/query-1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27.json b/backend/.sqlx/query-1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27.json new file mode 100644 index 0000000000..7f1618b3ad --- /dev/null +++ b/backend/.sqlx/query-1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_progress\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND asset_path = ANY($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "1d4fb43c4856679371abfa5eeea8b92697ae3640a2e680395c2958c5d8664e27" +} diff --git a/backend/.sqlx/query-1e044b8f64184393953dfeadbeda8af703060a604e68f32e714610513aa284eb.json b/backend/.sqlx/query-1e044b8f64184393953dfeadbeda8af703060a604e68f32e714610513aa284eb.json new file mode 100644 index 0000000000..b22608a039 --- /dev/null +++ b/backend/.sqlx/query-1e044b8f64184393953dfeadbeda8af703060a604e68f32e714610513aa284eb.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of FROM flow WHERE workspace_id = 'wm-fork-obo'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "1e044b8f64184393953dfeadbeda8af703060a604e68f32e714610513aa284eb" +} diff --git a/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json b/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json index 9149698131..84ce6720d6 100644 --- a/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json +++ b/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e.json b/backend/.sqlx/query-1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e.json new file mode 100644 index 0000000000..57861d6518 --- /dev/null +++ b/backend/.sqlx/query-1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1f39f7ca0c04e825d3648906eed0c98c1dc8c913dc95b691cc6f92ffcd57989e" +} diff --git a/backend/.sqlx/query-1fb5590cb1fe706b0d2dffcd04a41bb2fce6012b812d16d37c895b2eb9aeca4c.json b/backend/.sqlx/query-1fb5590cb1fe706b0d2dffcd04a41bb2fce6012b812d16d37c895b2eb9aeca4c.json new file mode 100644 index 0000000000..d58ac0c4e4 --- /dev/null +++ b/backend/.sqlx/query-1fb5590cb1fe706b0d2dffcd04a41bb2fce6012b812d16d37c895b2eb9aeca4c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n 'u/a/wh/analytics/draft', 'select 3', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1fb5590cb1fe706b0d2dffcd04a41bb2fce6012b812d16d37c895b2eb9aeca4c" +} diff --git a/backend/.sqlx/query-2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0.json b/backend/.sqlx/query-2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0.json deleted file mode 100644 index 14f17d871e..0000000000 --- a/backend/.sqlx/query-2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_settings\n SET\n ai_config = source_ws.ai_config,\n large_file_storage = source_ws.large_file_storage,\n ducklake = source_ws.ducklake,\n datatable = source_ws.datatable,\n git_app_installations = source_ws.git_app_installations\n FROM workspace_settings source_ws\n WHERE source_ws.workspace_id = $1\n AND workspace_settings.workspace_id = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "2213dc4b594d27b3788c48b19e13c600224681e11c016eea5cee3086a7bea8d0" -} diff --git a/backend/.sqlx/query-229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07.json b/backend/.sqlx/query-229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07.json new file mode 100644 index 0000000000..1e92b86bb2 --- /dev/null +++ b/backend/.sqlx/query-229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags, description,\n columns, freshness)\n VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders',\n 'u/a/wh/analytics/orders', 'select 1', '{finance}', 'daily order facts',\n '{\"order_id\": {\"description\": \"natural key\"}}'::jsonb,\n '{\"warn_after\": {\"count\": 12, \"period\": \"hour\"}}'::jsonb)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "229cc3975b7f579c35ac0daa9204ff3ff25fb57954dd1ba2d552c7b263696c07" +} diff --git a/backend/.sqlx/query-22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049.json b/backend/.sqlx/query-22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049.json new file mode 100644 index 0000000000..baef76a508 --- /dev/null +++ b/backend/.sqlx/query-22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH keep AS (\n SELECT hash FROM script\n WHERE workspace_id = $2 AND path = $3 AND language = 'dbt'\n ORDER BY created_at DESC LIMIT $1\n )\n DELETE FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $2 AND g.script_path = $3\n AND g.job_id = '00000000-0000-0000-0000-000000000000'\n AND NOT EXISTS (SELECT 1 FROM keep k WHERE k.hash = g.script_hash)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "22be2359b966bd92d8327f142281ec620fc98c5fb77061ed192a6f75bea2b049" +} diff --git a/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json b/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json index e3f72d9c3a..cfcb14ce72 100644 --- a/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json +++ b/backend/.sqlx/query-231475dc825518aa88f562698f8061d2562c3ac4af8d088f9eea9f0bc11d5fe7.json @@ -27,7 +27,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json index 02a419de06..16f26c4b78 100644 --- a/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json +++ b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json @@ -21,7 +21,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-295325886239353a27bfb8807a59952b81141a2544fc5340e6526b8fa0fb06c5.json b/backend/.sqlx/query-295325886239353a27bfb8807a59952b81141a2544fc5340e6526b8fa0fb06c5.json new file mode 100644 index 0000000000..1a5cb5dbce --- /dev/null +++ b/backend/.sqlx/query-295325886239353a27bfb8807a59952b81141a2544fc5340e6526b8fa0fb06c5.json @@ -0,0 +1,101 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels, on_behalf_of, on_behalf_of_email) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar", + "Int8Array", + "Text", + "Text", + "Text", + "Varchar", + "Text", + "Bool", + "Jsonb", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + }, + { + "Custom": { + "name": "script_kind", + "kind": { + "Enum": [ + "script", + "trigger", + "failure", + "command", + "approval", + "preprocessor" + ] + } + } + }, + "Varchar", + "VarcharArray", + "Int4", + "Int4", + "Int4", + "Bool", + "Bool", + "Int2", + "Bool", + "Bool", + "Int4", + "Int4", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Jsonb", + "Varchar", + "Int4", + "Bool", + "Int8", + "Jsonb", + "TextArray", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "295325886239353a27bfb8807a59952b81141a2544fc5340e6526b8fa0fb06c5" +} diff --git a/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json b/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json index f940f25b4a..99850586d3 100644 --- a/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json +++ b/backend/.sqlx/query-2a28f13c1f06a77bc9a164393cccf0d8d2a3d7e8552d8ad09512d2d6bf27c3fb.json @@ -42,7 +42,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-2a89648b28c40dfb9ebd4aa0b5ffa772d88c5cf97de840b5147deaeb95c98209.json b/backend/.sqlx/query-2a89648b28c40dfb9ebd4aa0b5ffa772d88c5cf97de840b5147deaeb95c98209.json new file mode 100644 index 0000000000..3ef757423e --- /dev/null +++ b/backend/.sqlx/query-2a89648b28c40dfb9ebd4aa0b5ffa772d88c5cf97de840b5147deaeb95c98209.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2a89648b28c40dfb9ebd4aa0b5ffa772d88c5cf97de840b5147deaeb95c98209" +} diff --git a/backend/.sqlx/query-2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680.json b/backend/.sqlx/query-2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680.json new file mode 100644 index 0000000000..f770ee58df --- /dev/null +++ b/backend/.sqlx/query-2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT relation_root_at_last_ingest FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3\n AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "relation_root_at_last_ingest", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + true + ] + }, + "hash": "2a8b9b199a6d383ed39a64dad54014f869ade4a139956dcb082f0524779f7680" +} diff --git a/backend/.sqlx/query-2c56da59ac2d7b410235a3181cf2db835776ff3928236dc26033a9f577ddad61.json b/backend/.sqlx/query-2c56da59ac2d7b410235a3181cf2db835776ff3928236dc26033a9f577ddad61.json new file mode 100644 index 0000000000..76c8a429fc --- /dev/null +++ b/backend/.sqlx/query-2c56da59ac2d7b410235a3181cf2db835776ff3928236dc26033a9f577ddad61.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS \"acquired!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "acquired!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2c56da59ac2d7b410235a3181cf2db835776ff3928236dc26033a9f577ddad61" +} diff --git a/backend/.sqlx/query-2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3.json b/backend/.sqlx/query-2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3.json new file mode 100644 index 0000000000..558e5e466e --- /dev/null +++ b/backend/.sqlx/query-2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3 AND retryable", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "2d943b22eb69010f3c2388b9443dc8c5729cb2b828cae94c047b896ebff37fd3" +} diff --git a/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json index ded7b65a83..346a46d150 100644 --- a/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json +++ b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json @@ -21,7 +21,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-31f38820514b1d51459ad93dd0e59d96d4e8cc79c8f3d3c4228f3bc7585c1839.json b/backend/.sqlx/query-31f38820514b1d51459ad93dd0e59d96d4e8cc79c8f3d3c4228f3bc7585c1839.json new file mode 100644 index 0000000000..69c34a9bc3 --- /dev/null +++ b/backend/.sqlx/query-31f38820514b1d51459ad93dd0e59d96d4e8cc79c8f3d3c4228f3bc7585c1839.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of)\n VALUES ('test-workspace', 'u/test-user/s', 93001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'ext@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "31f38820514b1d51459ad93dd0e59d96d4e8cc79c8f3d3c4228f3bc7585c1839" +} diff --git a/backend/.sqlx/query-33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28.json b/backend/.sqlx/query-33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28.json new file mode 100644 index 0000000000..8316ec5fab --- /dev/null +++ b/backend/.sqlx/query-33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_node WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "33b9ef6fabbb5f0de64246b51f0ce702603b7ae170054cda7ed5f6f0a9f44a28" +} diff --git a/backend/.sqlx/query-34a0dd9de9495f80a07e568323c5b94e33dfcacba9466bd870152b130d400794.json b/backend/.sqlx/query-34a0dd9de9495f80a07e568323c5b94e33dfcacba9466bd870152b130d400794.json new file mode 100644 index 0000000000..706486c23c --- /dev/null +++ b/backend/.sqlx/query-34a0dd9de9495f80a07e568323c5b94e33dfcacba9466bd870152b130d400794.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.runnable_path, j.runnable_id, j.permissioned_as\n FROM v2_job_queue q JOIN v2_job j ON j.id = q.id\n WHERE q.id = $1 AND j.workspace_id = $2 AND q.tag = ANY($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "permissioned_as", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + true, + true, + false + ] + }, + "hash": "34a0dd9de9495f80a07e568323c5b94e33dfcacba9466bd870152b130d400794" +} diff --git a/backend/.sqlx/query-359b85d7940b82bb0e40ebf09e054f9191672e92fff6b22c2130bb04c7009665.json b/backend/.sqlx/query-359b85d7940b82bb0e40ebf09e054f9191672e92fff6b22c2130bb04c7009665.json new file mode 100644 index 0000000000..e62e71a89a --- /dev/null +++ b/backend/.sqlx/query-359b85d7940b82bb0e40ebf09e054f9191672e92fff6b22c2130bb04c7009665.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, on_behalf_of, on_behalf_of_email FROM script WHERE workspace_id = 'wm-fork-obo' ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "on_behalf_of_email", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true, + true + ] + }, + "hash": "359b85d7940b82bb0e40ebf09e054f9191672e92fff6b22c2130bb04c7009665" +} diff --git a/backend/.sqlx/query-36d0ace456b6022e5311491e0ef457b33679b6ffe75d068349638f9345693348.json b/backend/.sqlx/query-36d0ace456b6022e5311491e0ef457b33679b6ffe75d068349638f9345693348.json new file mode 100644 index 0000000000..6cd95a08ac --- /dev/null +++ b/backend/.sqlx/query-36d0ace456b6022e5311491e0ef457b33679b6ffe75d068349638f9345693348.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET on_behalf_of = $1 WHERE on_behalf_of = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "36d0ace456b6022e5311491e0ef457b33679b6ffe75d068349638f9345693348" +} diff --git a/backend/.sqlx/query-38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df.json b/backend/.sqlx/query-38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df.json new file mode 100644 index 0000000000..a779a3c995 --- /dev/null +++ b/backend/.sqlx/query-38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_node WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "38125105da39bfedbdc1396fe37a8835f2630389dc339be0fda614ddb4d773df" +} diff --git a/backend/.sqlx/query-381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549.json b/backend/.sqlx/query-381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549.json new file mode 100644 index 0000000000..18df498bba --- /dev/null +++ b/backend/.sqlx/query-381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET archived = true, extra_perms = '{\"u/outsider\": true}'::jsonb\n WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "381d59e7ab08e2ae7147e4c277a2d96f22375fa2b1eccafb7f09928a8b0cb549" +} diff --git a/backend/.sqlx/query-38c20dba51b1e2b4b28e5eed51f9071e1d3cf78e89ec467927823177c5a439cc.json b/backend/.sqlx/query-38c20dba51b1e2b4b28e5eed51f9071e1d3cf78e89ec467927823177c5a439cc.json new file mode 100644 index 0000000000..66681c2377 --- /dev/null +++ b/backend/.sqlx/query-38c20dba51b1e2b4b28e5eed51f9071e1d3cf78e89ec467927823177c5a439cc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id FROM usr WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "38c20dba51b1e2b4b28e5eed51f9071e1d3cf78e89ec467927823177c5a439cc" +} diff --git a/backend/.sqlx/query-3b20f17e0fadb619de37cda93b361f73d964ae4562e0cb862c7402ae779ed13a.json b/backend/.sqlx/query-3b20f17e0fadb619de37cda93b361f73d964ae4562e0cb862c7402ae779ed13a.json new file mode 100644 index 0000000000..03b6042099 --- /dev/null +++ b/backend/.sqlx/query-3b20f17e0fadb619de37cda93b361f73d964ae4562e0cb862c7402ae779ed13a.json @@ -0,0 +1,153 @@ +{ + "db_name": "PostgreSQL", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, cache_ignore_s3_path, runnable_settings_handle, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of, created_by, labels FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false)\n ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "concurrency_key", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "debounce_key", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "debounce_delay_s", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 11, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 13, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 14, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "labels", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false, + true + ] + }, + "hash": "3b20f17e0fadb619de37cda93b361f73d964ae4562e0cb862c7402ae779ed13a" +} diff --git a/backend/.sqlx/query-3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d.json b/backend/.sqlx/query-3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d.json new file mode 100644 index 0000000000..d8afb95017 --- /dev/null +++ b/backend/.sqlx/query-3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders',\n 'u/a/wh/analytics/orders', 'select 2', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "3b40cccf059fbd28ff9c8d0004daee26c81a8fe909e12ae997eaaba83ade634d" +} diff --git a/backend/.sqlx/query-3c44de953b08ecfd8a962490d21b2dd0bfb66b7631acc958742523cce63793a0.json b/backend/.sqlx/query-3c44de953b08ecfd8a962490d21b2dd0bfb66b7631acc958742523cce63793a0.json new file mode 100644 index 0000000000..4a6a4f0cb6 --- /dev/null +++ b/backend/.sqlx/query-3c44de953b08ecfd8a962490d21b2dd0bfb66b7631acc958742523cce63793a0.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (\n workspace_id, path, summary, description,\n dependency_job, lock_error_logs, tag,\n dedicated_worker, visible_to_runner_only,\n ws_error_handler_muted,\n value, schema, edited_by, edited_at, labels,\n on_behalf_of, on_behalf_of_email\n ) VALUES (\n $1, $2, $3, $4,\n NULL, '', $5,\n $6, $7,\n $8,\n $9, $10::text::json, $11, now(), $12,\n $13, $14\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Bool", + "Bool", + "Bool", + "Jsonb", + "Text", + "Varchar", + "TextArray", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3c44de953b08ecfd8a962490d21b2dd0bfb66b7631acc958742523cce63793a0" +} diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index 257e1e528f..2a02e5a8ce 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -77,7 +77,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-3cc398db9c2f8f698a45cb763036c554a9d7525ad34f9ec1bd4ac3dc7afaee38.json b/backend/.sqlx/query-3cc398db9c2f8f698a45cb763036c554a9d7525ad34f9ec1bd4ac3dc7afaee38.json new file mode 100644 index 0000000000..3905a1c7d1 --- /dev/null +++ b/backend/.sqlx/query-3cc398db9c2f8f698a45cb763036c554a9d7525ad34f9ec1bd4ac3dc7afaee38.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (\n workspace_id, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, dependency_job, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, versions, on_behalf_of, on_behalf_of_email, lock_error_logs\n )\n SELECT $2, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, NULL, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, ARRAY[]::bigint[],\n -- Same predicate as clone_scripts.\n CASE WHEN on_behalf_of LIKE 'u/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $2::varchar\n AND u.username = substring(on_behalf_of from 3)\n UNION ALL\n SELECT 1 FROM password p WHERE p.super_admin\n AND (p.username = substring(on_behalf_of from 3)\n OR p.email = substring(on_behalf_of from 3))))\n WHEN on_behalf_of LIKE 'g/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM group_ g WHERE g.workspace_id = $2::varchar\n AND g.name = substring(on_behalf_of from 3)))\n ELSE\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $2::varchar\n AND u.username = on_behalf_of\n UNION ALL\n SELECT 1 FROM password p WHERE p.email = on_behalf_of\n AND p.super_admin))\n END, on_behalf_of_email, lock_error_logs\n FROM flow\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3cc398db9c2f8f698a45cb763036c554a9d7525ad34f9ec1bd4ac3dc7afaee38" +} diff --git a/backend/.sqlx/query-3e474475c05ff5fc137a070b149dd3a5a77e1640e03f2964d5143c1063fd3d83.json b/backend/.sqlx/query-3e474475c05ff5fc137a070b149dd3a5a77e1640e03f2964d5143c1063fd3d83.json new file mode 100644 index 0000000000..77d9c6d71b --- /dev/null +++ b/backend/.sqlx/query-3e474475c05ff5fc137a070b149dd3a5a77e1640e03f2964d5143c1063fd3d83.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[]) RETURNING datatable, timestamp, name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "3e474475c05ff5fc137a070b149dd3a5a77e1640e03f2964d5143c1063fd3d83" +} diff --git a/backend/.sqlx/query-2a510a8bec98055796f987d86c344ca116895d71de09db338ec09e425dcebe5e.json b/backend/.sqlx/query-3e869b95799300dff973a3c858f823a2872e33a360ff5a02169e4e80ce3d3b7b.json similarity index 75% rename from backend/.sqlx/query-2a510a8bec98055796f987d86c344ca116895d71de09db338ec09e425dcebe5e.json rename to backend/.sqlx/query-3e869b95799300dff973a3c858f823a2872e33a360ff5a02169e4e80ce3d3b7b.json index 23337708f8..ce3b4fcdf8 100644 --- a/backend/.sqlx/query-2a510a8bec98055796f987d86c344ca116895d71de09db338ec09e425dcebe5e.json +++ b/backend/.sqlx/query-3e869b95799300dff973a3c858f823a2872e33a360ff5a02169e4e80ce3d3b7b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, username, is_admin, is_operator, groups, folders FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + "query": "SELECT email, username, is_admin, is_operator, groups, folders, end_user_email FROM job_perms WHERE job_id = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "folders", "type_info": "JsonbArray" + }, + { + "ordinal": 6, + "name": "end_user_email", + "type_info": "Varchar" } ], "parameters": { @@ -46,8 +51,9 @@ false, false, false, - false + false, + true ] }, - "hash": "2a510a8bec98055796f987d86c344ca116895d71de09db338ec09e425dcebe5e" + "hash": "3e869b95799300dff973a3c858f823a2872e33a360ff5a02169e4e80ce3d3b7b" } diff --git a/backend/.sqlx/query-3ec280ad74cf63dc3cebb312c620e7a5c2a3b75051618fc3c9635b4cd8d48e25.json b/backend/.sqlx/query-3ec280ad74cf63dc3cebb312c620e7a5c2a3b75051618fc3c9635b4cd8d48e25.json new file mode 100644 index 0000000000..7cbc8115c9 --- /dev/null +++ b/backend/.sqlx/query-3ec280ad74cf63dc3cebb312c620e7a5c2a3b75051618fc3c9635b4cd8d48e25.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3ec280ad74cf63dc3cebb312c620e7a5c2a3b75051618fc3c9635b4cd8d48e25" +} diff --git a/backend/.sqlx/query-407fdc870c39138811beb17e125bff3946d829f3d482a9b732ae3b37a9f7f2b5.json b/backend/.sqlx/query-407fdc870c39138811beb17e125bff3946d829f3d482a9b732ae3b37a9f7f2b5.json new file mode 100644 index 0000000000..3db0bad5a7 --- /dev/null +++ b/backend/.sqlx/query-407fdc870c39138811beb17e125bff3946d829f3d482a9b732ae3b37a9f7f2b5.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "407fdc870c39138811beb17e125bff3946d829f3d482a9b732ae3b37a9f7f2b5" +} diff --git a/backend/.sqlx/query-41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b.json b/backend/.sqlx/query-41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b.json new file mode 100644 index 0000000000..c812a91912 --- /dev/null +++ b/backend/.sqlx/query-41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_progress\n WHERE workspace_id = $1 AND updated_at < now() - make_interval(days => $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "41428ecb8940a8fb4043b71eacb554314177fc11bc82212d95182e3723e64c6b" +} diff --git a/backend/.sqlx/query-42f2cf3165850524804842c51965c4020193a613357c7449644e09a89d6d5346.json b/backend/.sqlx/query-42f2cf3165850524804842c51965c4020193a613357c7449644e09a89d6d5346.json new file mode 100644 index 0000000000..45031a9f99 --- /dev/null +++ b/backend/.sqlx/query-42f2cf3165850524804842c51965c4020193a613357c7449644e09a89d6d5346.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot\n (workspace_id, script_path, script_hash, job_id, permissioned_as, digest, ingested_at)\n VALUES ($1, $2, NULL, $3, $4, $5, now())", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "42f2cf3165850524804842c51965c4020193a613357c7449644e09a89d6d5346" +} diff --git a/backend/.sqlx/query-4301a7ad1f7c09e26f5dc7044e0a270be78e44c167fe3f6a73e0ff4b1e3baa35.json b/backend/.sqlx/query-4301a7ad1f7c09e26f5dc7044e0a270be78e44c167fe3f6a73e0ff4b1e3baa35.json new file mode 100644 index 0000000000..3c41b3fb85 --- /dev/null +++ b/backend/.sqlx/query-4301a7ad1f7c09e26f5dc7044e0a270be78e44c167fe3f6a73e0ff4b1e3baa35.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, on_behalf_of)\n VALUES ('test-workspace', 'u/test-user/obo_flow', '', '', $1, 'test-user', NOW(), 'u/test-user-2')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "4301a7ad1f7c09e26f5dc7044e0a270be78e44c167fe3f6a73e0ff4b1e3baa35" +} diff --git a/backend/.sqlx/query-4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c.json b/backend/.sqlx/query-4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c.json new file mode 100644 index 0000000000..4762ed3279 --- /dev/null +++ b/backend/.sqlx/query-4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_state SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4330e66917f04dc6597d323e9102de473e3f319ba13fa0784dd5bef24224233c" +} diff --git a/backend/.sqlx/query-4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e.json b/backend/.sqlx/query-4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e.json new file mode 100644 index 0000000000..a2e8f4127c --- /dev/null +++ b/backend/.sqlx/query-4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content,\n created_by, language, lock)\n VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "4374167ebf64c4e8dd683e31451800c2d068cbafbd279454724451d7f57e108e" +} diff --git a/backend/.sqlx/query-471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339.json b/backend/.sqlx/query-471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339.json new file mode 100644 index 0000000000..5903661c1a --- /dev/null +++ b/backend/.sqlx/query-471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_id FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "471edf439cb293aa824440a2ef2f0afc28f425b061e90e1dfc6133b426a7f339" +} diff --git a/backend/.sqlx/query-48969b648d36da8ef422d8f936eb5324655def916a0ac6f6a65eb649b17f102d.json b/backend/.sqlx/query-48969b648d36da8ef422d8f936eb5324655def916a0ac6f6a65eb649b17f102d.json new file mode 100644 index 0000000000..ba9ccbbb77 --- /dev/null +++ b/backend/.sqlx/query-48969b648d36da8ef422d8f936eb5324655def916a0ac6f6a65eb649b17f102d.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id\n WHERE w.deleted = false AND w.id NOT LIKE 'wm-fork%'\n AND jsonb_exists(ws.large_file_storage->'secondary_storage', 'main')\n ) AS \"exists!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "48969b648d36da8ef422d8f936eb5324655def916a0ac6f6a65eb649b17f102d" +} diff --git a/backend/.sqlx/query-493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91.json b/backend/.sqlx/query-493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91.json new file mode 100644 index 0000000000..2f36ff7969 --- /dev/null +++ b/backend/.sqlx/query-493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET dbt_warehouses = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "493b29e093bb2b8f8ed6947843e26fe8fb6aefe54ad5862cad9b8fd9868cdd91" +} diff --git a/backend/.sqlx/query-4956c7bb3312520e0ba0ae330037056f9a6157c1c448e46f502a17615ec25d87.json b/backend/.sqlx/query-4956c7bb3312520e0ba0ae330037056f9a6157c1c448e46f502a17615ec25d87.json new file mode 100644 index 0000000000..c43bb800bb --- /dev/null +++ b/backend/.sqlx/query-4956c7bb3312520e0ba0ae330037056f9a6157c1c448e46f502a17615ec25d87.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH t1 AS (UPDATE http_trigger SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), t2 AS (UPDATE email_trigger SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4) UPDATE native_trigger SET script_path = $1, updated_at = NOW(), error = $5 WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4 RETURNING service_name::text AS \"service_name!\", external_id, script_path, is_flow", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "service_name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "external_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null, + false, + false, + false + ] + }, + "hash": "4956c7bb3312520e0ba0ae330037056f9a6157c1c448e46f502a17615ec25d87" +} diff --git a/backend/.sqlx/query-4e3f3202e762aad38707315c05c9c3b7cebdbd9be8b5cc9a14b325c4ad1f5b4f.json b/backend/.sqlx/query-4e3f3202e762aad38707315c05c9c3b7cebdbd9be8b5cc9a14b325c4ad1f5b4f.json new file mode 100644 index 0000000000..cf360c9dd5 --- /dev/null +++ b/backend/.sqlx/query-4e3f3202e762aad38707315c05c9c3b7cebdbd9be8b5cc9a14b325c4ad1f5b4f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_lock(hashtextextended($1, 0))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4e3f3202e762aad38707315c05c9c3b7cebdbd9be8b5cc9a14b325c4ad1f5b4f" +} diff --git a/backend/.sqlx/query-4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61.json b/backend/.sqlx/query-4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61.json new file mode 100644 index 0000000000..ee7535ffc9 --- /dev/null +++ b/backend/.sqlx/query-4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT dbt_warehouses FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "dbt_warehouses", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "4ed961992cd6c99e2efe889ab1cf06e6ad12305a5b845948dc200f304fb87a61" +} diff --git a/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json b/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json index a7f5d2f785..e4e2c240c7 100644 --- a/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json +++ b/backend/.sqlx/query-50f81c128d644c60837e603099e169fbc7a500c11e79d2c78d6eccccf6491aec.json @@ -71,7 +71,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json b/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json index 6a010dd1f4..f1941c012c 100644 --- a/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json +++ b/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json @@ -35,7 +35,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json b/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json index c617944b91..b00c398445 100644 --- a/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json +++ b/backend/.sqlx/query-5387fbfd4be55674dcbe9f9b3ce9649d0f5db125776fbdffea993a53a09708de.json @@ -41,7 +41,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8.json b/backend/.sqlx/query-56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8.json new file mode 100644 index 0000000000..b467a2c39b --- /dev/null +++ b/backend/.sqlx/query-56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, child_unique_id, ingested_at)\n SELECT $2, script_path, script_hash, job_id, parent_unique_id, child_unique_id,\n ingested_at\n FROM dbt_edge\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "56badc145d03676b6bd80a3830ce0a0401517a9b60e5689d3c9cdc9a87aaf2a8" +} diff --git a/backend/.sqlx/query-58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a.json b/backend/.sqlx/query-58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a.json new file mode 100644 index 0000000000..1edc69b002 --- /dev/null +++ b/backend/.sqlx/query-58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT result->'materialized' FROM v2_job_completed WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58e7d590631294899025c2cc279c4352708f65a6c094a89b4cafa1e6450a4e6a" +} diff --git a/backend/.sqlx/query-59a913a29c3f32d2d0771f46f0d408e686283c8a6737ad4ededa019158e2cb8d.json b/backend/.sqlx/query-59a913a29c3f32d2d0771f46f0d408e686283c8a6737ad4ededa019158e2cb8d.json new file mode 100644 index 0000000000..b89b1ff7f9 --- /dev/null +++ b/backend/.sqlx/query-59a913a29c3f32d2d0771f46f0d408e686283c8a6737ad4ededa019158e2cb8d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name)\n VALUES ($1, '', 'password', true, true, '')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "59a913a29c3f32d2d0771f46f0d408e686283c8a6737ad4ededa019158e2cb8d" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5a4e42f3bfa96bb1184adeccf14580c46c1e77cb8dd9b36523965d79eb5416bb.json b/backend/.sqlx/query-5a4e42f3bfa96bb1184adeccf14580c46c1e77cb8dd9b36523965d79eb5416bb.json new file mode 100644 index 0000000000..c1f6558712 --- /dev/null +++ b/backend/.sqlx/query-5a4e42f3bfa96bb1184adeccf14580c46c1e77cb8dd9b36523965d79eb5416bb.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of, on_behalf_of_email)\n VALUES ('test-workspace', 'u/test-user/sg', 95001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'g/ops', 'group-ops@windmill.dev'),\n ('test-workspace', 'u/test-user/su', 95002, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2', 'group-ops@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5a4e42f3bfa96bb1184adeccf14580c46c1e77cb8dd9b36523965d79eb5416bb" +} diff --git a/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json index c126b6371a..40c66a93f9 100644 --- a/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json +++ b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } @@ -71,7 +72,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-5c0d0c7111b70b119b3182c9195361388ead36dc6ab03c6a7c17fb97d2b60a67.json b/backend/.sqlx/query-5c0d0c7111b70b119b3182c9195361388ead36dc6ab03c6a7c17fb97d2b60a67.json index e72722d53e..be7d99f4a6 100644 --- a/backend/.sqlx/query-5c0d0c7111b70b119b3182c9195361388ead36dc6ab03c6a7c17fb97d2b60a67.json +++ b/backend/.sqlx/query-5c0d0c7111b70b119b3182c9195361388ead36dc6ab03c6a7c17fb97d2b60a67.json @@ -34,7 +34,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json index 03de8472ad..fc8ed63f51 100644 --- a/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json +++ b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json @@ -21,7 +21,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-5deb93bef71327582d997f3681b60e79b82f57c9c15c0870539ff057931c8f0a.json b/backend/.sqlx/query-5deb93bef71327582d997f3681b60e79b82f57c9c15c0870539ff057931c8f0a.json new file mode 100644 index 0000000000..01f7f523ed --- /dev/null +++ b/backend/.sqlx/query-5deb93bef71327582d997f3681b60e79b82f57c9c15c0870539ff057931c8f0a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5deb93bef71327582d997f3681b60e79b82f57c9c15c0870539ff057931c8f0a" +} diff --git a/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json b/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json index 8ff04d2fc1..34f1870b7a 100644 --- a/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json +++ b/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-5f95454c29587117f7a482ca74506cda7ee52f5a85aea5c2186353f8b1decc46.json b/backend/.sqlx/query-5f95454c29587117f7a482ca74506cda7ee52f5a85aea5c2186353f8b1decc46.json new file mode 100644 index 0000000000..d1b9e3c858 --- /dev/null +++ b/backend/.sqlx/query-5f95454c29587117f7a482ca74506cda7ee52f5a85aea5c2186353f8b1decc46.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "5f95454c29587117f7a482ca74506cda7ee52f5a85aea5c2186353f8b1decc46" +} diff --git a/backend/.sqlx/query-5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523.json b/backend/.sqlx/query-5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523.json new file mode 100644 index 0000000000..418083acf9 --- /dev/null +++ b/backend/.sqlx/query-5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT script_lang = 'dbt' FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5fd6467e3316b64e0451614d6bb9101644d1869af85687ab55067308644a2523" +} diff --git a/backend/.sqlx/query-608d487e6f3313d57c1f2f043b6942c9ab6f58f9d7ffecb13a9de2c71b9bb709.json b/backend/.sqlx/query-608d487e6f3313d57c1f2f043b6942c9ab6f58f9d7ffecb13a9de2c71b9bb709.json new file mode 100644 index 0000000000..69951a8cb2 --- /dev/null +++ b/backend/.sqlx/query-608d487e6f3313d57c1f2f043b6942c9ab6f58f9d7ffecb13a9de2c71b9bb709.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, on_behalf_of FROM script WHERE workspace_id = 'wm-fork-obo' ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "on_behalf_of", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "608d487e6f3313d57c1f2f043b6942c9ab6f58f9d7ffecb13a9de2c71b9bb709" +} diff --git a/backend/.sqlx/query-617c19043d445ab1366630633e6afbeeea28b1d17dfacecb30dde65304cd9799.json b/backend/.sqlx/query-617c19043d445ab1366630633e6afbeeea28b1d17dfacecb30dde65304cd9799.json new file mode 100644 index 0000000000..9f41bf73e0 --- /dev/null +++ b/backend/.sqlx/query-617c19043d445ab1366630633e6afbeeea28b1d17dfacecb30dde65304cd9799.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO azure_trigger (\n azure_resource_path, azure_mode, scope_resource_id, topic_name,\n subscription_name, event_type_filters, push_auth_config, path, script_path,\n is_flow, workspace_id, edited_by, edited_at, extra_perms, server_id,\n last_server_ping, error, mode, permissioned_as, error_handler_path,\n error_handler_args, retry, labels\n )\n SELECT\n azure_resource_path, azure_mode, scope_resource_id, topic_name,\n subscription_name, event_type_filters, push_auth_config, path, script_path,\n is_flow, $1, edited_by, edited_at, extra_perms, NULL,\n NULL, NULL, 'disabled'::TRIGGER_MODE, permissioned_as, error_handler_path,\n error_handler_args, retry, labels\n FROM azure_trigger WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "617c19043d445ab1366630633e6afbeeea28b1d17dfacecb30dde65304cd9799" +} diff --git a/backend/.sqlx/query-6348061d79b0b6b25bda7425f7a37344e04107f060f949cc197b4df9bca18fb8.json b/backend/.sqlx/query-6348061d79b0b6b25bda7425f7a37344e04107f060f949cc197b4df9bca18fb8.json index f3d518eff0..3f1729e29a 100644 --- a/backend/.sqlx/query-6348061d79b0b6b25bda7425f7a37344e04107f060f949cc197b4df9bca18fb8.json +++ b/backend/.sqlx/query-6348061d79b0b6b25bda7425f7a37344e04107f060f949cc197b4df9bca18fb8.json @@ -34,7 +34,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-652c7a8adefc03f06485a7e223f54ebcab136711b89671192f379bf06bf3ca9b.json b/backend/.sqlx/query-652c7a8adefc03f06485a7e223f54ebcab136711b89671192f379bf06bf3ca9b.json new file mode 100644 index 0000000000..43a2c8952d --- /dev/null +++ b/backend/.sqlx/query-652c7a8adefc03f06485a7e223f54ebcab136711b89671192f379bf06bf3ca9b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username)\n VALUES ('sa@windmill.dev', 'not-a-real-hash', 'password', true, true, 'Ext', 'ext-sa')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "652c7a8adefc03f06485a7e223f54ebcab136711b89671192f379bf06bf3ca9b" +} diff --git a/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json b/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json index 80f172aa40..f6e000ba6f 100644 --- a/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json +++ b/backend/.sqlx/query-66c775b6e1120c5ed53b903d252e07f0965efb65c51580c529e61d09b8742dfd.json @@ -45,7 +45,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-680f38ee8b7fd09aa3d0f521bce5ff3c9bf42c7a51aadbe7d7680827989211d0.json b/backend/.sqlx/query-680f38ee8b7fd09aa3d0f521bce5ff3c9bf42c7a51aadbe7d7680827989211d0.json new file mode 100644 index 0000000000..22dfb381a0 --- /dev/null +++ b/backend/.sqlx/query-680f38ee8b7fd09aa3d0f521bce5ff3c9bf42c7a51aadbe7d7680827989211d0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND deleted = false\n ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "680f38ee8b7fd09aa3d0f521bce5ff3c9bf42c7a51aadbe7d7680827989211d0" +} diff --git a/backend/.sqlx/query-68aebbc8a7798abab3163aaffe298f6ea2722c68efff0c2a1baaf5de4fc247b2.json b/backend/.sqlx/query-68aebbc8a7798abab3163aaffe298f6ea2722c68efff0c2a1baaf5de4fc247b2.json new file mode 100644 index 0000000000..0b19784e3a --- /dev/null +++ b/backend/.sqlx/query-68aebbc8a7798abab3163aaffe298f6ea2722c68efff0c2a1baaf5de4fc247b2.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of, on_behalf_of_email)\n VALUES\n ('test-workspace', 'u/test-user/obo_member', 91001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user', 'test@windmill.dev'),\n ('test-workspace', 'u/test-user/obo_stranger', 91002, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2', 'test2@windmill.dev'),\n ('test-workspace', 'u/test-user/obo_group', 91003, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'g/all', 'group-all@windmill.dev'),\n ('test-workspace', 'u/test-user/obo_superadmin', 91004, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/ext-sa', 'sa@windmill.dev'),\n ('test-workspace', 'u/test-user/obo_address_only', 91005, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), NULL, 'test2@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "68aebbc8a7798abab3163aaffe298f6ea2722c68efff0c2a1baaf5de4fc247b2" +} diff --git a/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json index 144bcb13f2..b54e73c157 100644 --- a/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json +++ b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json @@ -22,7 +22,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e.json b/backend/.sqlx/query-6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e.json new file mode 100644 index 0000000000..8e6b0372b1 --- /dev/null +++ b/backend/.sqlx/query-6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT asset_kind AS \"asset_kind: windmill_common::assets::AssetKind\", asset_path,\n status::text AS \"status!\", row_count, error\n FROM dbt_run_progress\n WHERE workspace_id = $1 AND job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume", + "dbt" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "row_count", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false, + null, + true, + true + ] + }, + "hash": "6d89436b268157453784087c076ef5fb7c5ae46bede607cca308e1d5778e6d1e" +} diff --git a/backend/.sqlx/query-6f1489b2c52e08566ffa9450709df2b9404a30197b05554a868bd06b1ed8ccaa.json b/backend/.sqlx/query-6f1489b2c52e08566ffa9450709df2b9404a30197b05554a868bd06b1ed8ccaa.json new file mode 100644 index 0000000000..e71f76301e --- /dev/null +++ b/backend/.sqlx/query-6f1489b2c52e08566ffa9450709df2b9404a30197b05554a868bd06b1ed8ccaa.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6f1489b2c52e08566ffa9450709df2b9404a30197b05554a868bd06b1ed8ccaa" +} diff --git a/backend/.sqlx/query-717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0.json b/backend/.sqlx/query-717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0.json new file mode 100644 index 0000000000..6dc8e9acc2 --- /dev/null +++ b/backend/.sqlx/query-717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_node WHERE workspace_id = $1 AND script_hash = $2 AND job_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "717188b993d9c87f2de51d8edbafec074f8e02408efc6da2c742d81fd47c14f0" +} diff --git a/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json b/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json index 96a68f827d..64c6611843 100644 --- a/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json +++ b/backend/.sqlx/query-71767e6684957af5dff56a1bb64f980f712f92798d11feaeb0832962ad9ddb2e.json @@ -84,7 +84,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f.json b/backend/.sqlx/query-71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f.json new file mode 100644 index 0000000000..397b310807 --- /dev/null +++ b/backend/.sqlx/query-71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE materialized_partition\n SET status = 'failed',\n error = COALESCE(error, 'the run ended before this model finished')\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND NOT (asset_path = ANY($3))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "71b341ba06d588a7fb425d97921f16a8fc4fdf0bcebecfd21c9b7154dedea25f" +} diff --git a/backend/.sqlx/query-72fa27780070363e092a19dbea4b63195863245bfb271a946d2aa67c59da7b3c.json b/backend/.sqlx/query-72fa27780070363e092a19dbea4b63195863245bfb271a946d2aa67c59da7b3c.json new file mode 100644 index 0000000000..d97f8be4ac --- /dev/null +++ b/backend/.sqlx/query-72fa27780070363e092a19dbea4b63195863245bfb271a946d2aa67c59da7b3c.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, workspace_id, super_admin, owner FROM token WHERE token_hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "owner", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true, + false, + true + ] + }, + "hash": "72fa27780070363e092a19dbea4b63195863245bfb271a946d2aa67c59da7b3c" +} diff --git a/backend/.sqlx/query-74cab23105c5892b5df86ba1e81ac3d575db3d3542341988917f68165174f133.json b/backend/.sqlx/query-74cab23105c5892b5df86ba1e81ac3d575db3d3542341988917f68165174f133.json new file mode 100644 index 0000000000..a58ee643d0 --- /dev/null +++ b/backend/.sqlx/query-74cab23105c5892b5df86ba1e81ac3d575db3d3542341988917f68165174f133.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of'], to_jsonb($1::text))) WHERE value->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "74cab23105c5892b5df86ba1e81ac3d575db3d3542341988917f68165174f133" +} diff --git a/backend/.sqlx/query-77146f3ebe59007b7608c88ba35dddc63e9fde5b301e7f935def7e1d0d41e672.json b/backend/.sqlx/query-77146f3ebe59007b7608c88ba35dddc63e9fde5b301e7f935def7e1d0d41e672.json new file mode 100644 index 0000000000..35a2e8deda --- /dev/null +++ b/backend/.sqlx/query-77146f3ebe59007b7608c88ba35dddc63e9fde5b301e7f935def7e1d0d41e672.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT label, expiration, scopes FROM token WHERE token_hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "expiration", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "77146f3ebe59007b7608c88ba35dddc63e9fde5b301e7f935def7e1d0d41e672" +} diff --git a/backend/.sqlx/query-7ac16d499a630454d9327304d30ad667529f720e2a570b664679564cacec6708.json b/backend/.sqlx/query-7ac16d499a630454d9327304d30ad667529f720e2a570b664679564cacec6708.json new file mode 100644 index 0000000000..22182b5470 --- /dev/null +++ b/backend/.sqlx/query-7ac16d499a630454d9327304d30ad667529f720e2a570b664679564cacec6708.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at, labels)\n SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at, labels\n FROM flow\n WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ac16d499a630454d9327304d30ad667529f720e2a570b664679564cacec6708" +} diff --git a/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json b/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json index b38e22ad67..d26e9f7c87 100644 --- a/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json +++ b/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json @@ -77,7 +77,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce.json b/backend/.sqlx/query-7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce.json new file mode 100644 index 0000000000..936a85b611 --- /dev/null +++ b/backend/.sqlx/query-7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT identity, args, run_results, job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "identity", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "args", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "7be4645fe51c8a9bd13645a245846cb485c593b6df2194ae91ba2d8c584c0cce" +} diff --git a/backend/.sqlx/query-7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77.json b/backend/.sqlx/query-7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77.json new file mode 100644 index 0000000000..ee215b9615 --- /dev/null +++ b/backend/.sqlx/query-7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_graph_snapshot WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "7c19d92d25b799e8872452c5c77ae110e2c0628b2721a44735f4fc06e59b5c77" +} diff --git a/backend/.sqlx/query-7cddbc01dbec7878e7a8e528f21fbb75b10af4af97f5ecbe094db8f8080bf21d.json b/backend/.sqlx/query-7cddbc01dbec7878e7a8e528f21fbb75b10af4af97f5ecbe094db8f8080bf21d.json new file mode 100644 index 0000000000..72093610da --- /dev/null +++ b/backend/.sqlx/query-7cddbc01dbec7878e7a8e528f21fbb75b10af4af97f5ecbe094db8f8080bf21d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at)\n SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7cddbc01dbec7878e7a8e528f21fbb75b10af4af97f5ecbe094db8f8080bf21d" +} diff --git a/backend/.sqlx/query-7ce4e7af3d3cd01e644b8f1cca15db49ab38ac4982c7c258b17bf9ef53047cb6.json b/backend/.sqlx/query-7ce4e7af3d3cd01e644b8f1cca15db49ab38ac4982c7c258b17bf9ef53047cb6.json new file mode 100644 index 0000000000..04f84cd931 --- /dev/null +++ b/backend/.sqlx/query-7ce4e7af3d3cd01e644b8f1cca15db49ab38ac4982c7c258b17bf9ef53047cb6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow SET on_behalf_of = $1 WHERE on_behalf_of = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ce4e7af3d3cd01e644b8f1cca15db49ab38ac4982c7c258b17bf9ef53047cb6" +} diff --git a/backend/.sqlx/query-7e9e07f55ebb12d76efee5c92e98b85dbf7eebfdd64a605569b9b2f814f7fc77.json b/backend/.sqlx/query-7e9e07f55ebb12d76efee5c92e98b85dbf7eebfdd64a605569b9b2f814f7fc77.json new file mode 100644 index 0000000000..cc4dd630cf --- /dev/null +++ b/backend/.sqlx/query-7e9e07f55ebb12d76efee5c92e98b85dbf7eebfdd64a605569b9b2f814f7fc77.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs)\n SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs\n FROM flow WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7e9e07f55ebb12d76efee5c92e98b85dbf7eebfdd64a605569b9b2f814f7fc77" +} diff --git a/backend/.sqlx/query-8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78.json b/backend/.sqlx/query-8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78.json deleted file mode 100644 index 5453ec6899..0000000000 --- a/backend/.sqlx/query-8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "8058042d5564a934534517e9c0446c5fcc0c3654746c5e65994123ffb4d05f78" -} diff --git a/backend/.sqlx/query-836297c7d83185ac4f317e0bd0b2e73c55932927b05f963d45479103e3191ecd.json b/backend/.sqlx/query-836297c7d83185ac4f317e0bd0b2e73c55932927b05f963d45479103e3191ecd.json new file mode 100644 index 0000000000..a4026be742 --- /dev/null +++ b/backend/.sqlx/query-836297c7d83185ac4f317e0bd0b2e73c55932927b05f963d45479103e3191ecd.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "836297c7d83185ac4f317e0bd0b2e73c55932927b05f963d45479103e3191ecd" +} diff --git a/backend/.sqlx/query-83c4fed5d1a05365c3f486543519963c1068ea26e11307acec28fd07d0ce6b90.json b/backend/.sqlx/query-83c4fed5d1a05365c3f486543519963c1068ea26e11307acec28fd07d0ce6b90.json new file mode 100644 index 0000000000..baf02539cf --- /dev/null +++ b/backend/.sqlx/query-83c4fed5d1a05365c3f486543519963c1068ea26e11307acec28fd07d0ce6b90.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH inserted AS (\n INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs)\n SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM inserted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "83c4fed5d1a05365c3f486543519963c1068ea26e11307acec28fd07d0ce6b90" +} diff --git a/backend/.sqlx/query-843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc.json b/backend/.sqlx/query-843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc.json new file mode 100644 index 0000000000..5c000ffd34 --- /dev/null +++ b/backend/.sqlx/query-843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_graph_snapshot WHERE workspace_id = $1 AND job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "843ef7d1c619d2784306fa1dd06036c6967b5a2ebb88cf174f435a2ce692c5dc" +} diff --git a/backend/.sqlx/query-850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f.json b/backend/.sqlx/query-850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f.json new file mode 100644 index 0000000000..4c72eca991 --- /dev/null +++ b/backend/.sqlx/query-850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_graph_snapshot SET relation_root_at_last_ingest = $4\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3\n AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "850b5e3414851e53f9d5580b0ab1fdb97b053620ac14b8196ebf09208510fb1f" +} diff --git a/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json index 5aacf1a295..a535cf8aed 100644 --- a/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json +++ b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-86c1c8712587acbacc830fde1be608f7f9b84c13d8d906113467c00d440a8fa2.json b/backend/.sqlx/query-86c1c8712587acbacc830fde1be608f7f9b84c13d8d906113467c00d440a8fa2.json new file mode 100644 index 0000000000..3c4815172c --- /dev/null +++ b/backend/.sqlx/query-86c1c8712587acbacc830fde1be608f7f9b84c13d8d906113467c00d440a8fa2.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE native_trigger\n SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4,\n summary = $5, error = NULL, updated_at = NOW()\n WHERE\n workspace_id = $6\n AND service_name = $7\n AND external_id = $8\n AND script_path = $9\n AND is_flow = $10\n RETURNING 1 AS \"applied!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "applied!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Bool", + "Varchar", + "Jsonb", + "Varchar", + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + null + ] + }, + "hash": "86c1c8712587acbacc830fde1be608f7f9b84c13d8d906113467c00d440a8fa2" +} diff --git a/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json b/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json new file mode 100644 index 0000000000..8cdb2d5907 --- /dev/null +++ b/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, path, typ, value, created_at, email)\n SELECT $2, path, typ,\n CASE WHEN typ IN ('script', 'flow')\n THEN to_json(to_jsonb(value) - 'on_behalf_of')\n ELSE value END,\n created_at, email\n FROM draft\n WHERE workspace_id = $1 AND (email = $3 OR email IS NULL)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5" +} diff --git a/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json b/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json index ffce4491e3..f26bca1232 100644 --- a/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json +++ b/backend/.sqlx/query-87ca1f2a34f54dade76f5a2cde5ecdac6806e1a306dca880943d97bb6d6a889d.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-8901dc69384685a7e05f39e1fae65471decea3d46ee35582c3b7c1ac333d8dd5.json b/backend/.sqlx/query-8901dc69384685a7e05f39e1fae65471decea3d46ee35582c3b7c1ac333d8dd5.json new file mode 100644 index 0000000000..e2b87fd182 --- /dev/null +++ b/backend/.sqlx/query-8901dc69384685a7e05f39e1fae65471decea3d46ee35582c3b7c1ac333d8dd5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(\n SELECT 1 FROM script WHERE on_behalf_of = $1\n UNION ALL SELECT 1 FROM flow WHERE on_behalf_of = $1\n UNION ALL SELECT 1 FROM app WHERE policy->>'on_behalf_of' = $1\n UNION ALL SELECT 1 FROM schedule WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM http_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM websocket_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM postgres_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM mqtt_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM kafka_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM nats_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM sqs_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM gcp_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM email_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM amqp_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM azure_trigger WHERE permissioned_as = $1\n UNION ALL SELECT 1 FROM folder\n WHERE default_permissioned_as @> jsonb_build_array(\n jsonb_build_object('permissioned_as', $1::text)))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8901dc69384685a7e05f39e1fae65471decea3d46ee35582c3b7c1ac333d8dd5" +} diff --git a/backend/.sqlx/query-8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166.json b/backend/.sqlx/query-8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166.json new file mode 100644 index 0000000000..0f743221dc --- /dev/null +++ b/backend/.sqlx/query-8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language, lock)\n VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "8a40b3e7b9c59f1a0b50d3fe6b320748a912eff01dda22df9f489b6703435166" +} diff --git a/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json b/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json index 6f7468033a..44e6cc5c81 100644 --- a/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json +++ b/backend/.sqlx/query-8acc8c47123af84d2308042dcd34d04b941e928c7621e66d3bdc510c335f8c20.json @@ -42,7 +42,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280.json b/backend/.sqlx/query-8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280.json new file mode 100644 index 0000000000..a087c3619f --- /dev/null +++ b/backend/.sqlx/query-8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_progress SET workspace_id = $1\n WHERE workspace_id = $2 AND job_id IN (SELECT id FROM v2_job WHERE workspace_id = $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8bad2a916f9afe1c037a01ffe69e136b562ace2d8265c9d26bcdf866f0f2c280" +} diff --git a/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json index 24d70efae1..09fc3c9ad1 100644 --- a/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json +++ b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json @@ -36,7 +36,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081.json b/backend/.sqlx/query-8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081.json index d7f59025e7..fbb3d579de 100644 --- a/backend/.sqlx/query-8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081.json +++ b/backend/.sqlx/query-8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081.json @@ -154,7 +154,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e.json b/backend/.sqlx/query-8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e.json new file mode 100644 index 0000000000..04d2fe0a77 --- /dev/null +++ b/backend/.sqlx/query-8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_graph_snapshot WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8e4397a299c29687bec108899bd520d67718b2c60e717d69439ea13da272c81e" +} diff --git a/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json b/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json index eac4547361..406917a99f 100644 --- a/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json +++ b/backend/.sqlx/query-8e829a7358ea47100c99e78067266eb7a83e312b88f0244d1c517b03a9bcdea0.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-90489615820818332e40a46a3d720977b23f696e74735de1a56da58713e64b50.json b/backend/.sqlx/query-90489615820818332e40a46a3d720977b23f696e74735de1a56da58713e64b50.json new file mode 100644 index 0000000000..ebc76b2fb4 --- /dev/null +++ b/backend/.sqlx/query-90489615820818332e40a46a3d720977b23f696e74735de1a56da58713e64b50.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET path = $1 WHERE workspace_id = 'test-workspace' AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "90489615820818332e40a46a3d720977b23f696e74735de1a56da58713e64b50" +} diff --git a/backend/.sqlx/query-92723ff15a596e97edc71d24a5b653deb9d0e44259b22f2c6c5f43579f687360.json b/backend/.sqlx/query-92723ff15a596e97edc71d24a5b653deb9d0e44259b22f2c6c5f43579f687360.json new file mode 100644 index 0000000000..860da63f0b --- /dev/null +++ b/backend/.sqlx/query-92723ff15a596e97edc71d24a5b653deb9d0e44259b22f2c6c5f43579f687360.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (\n workspace_id, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n on_behalf_of, on_behalf_of_email, assets, modules\n )\n SELECT\n $1, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n -- Same three forms and the same prefix-first rule as permissioned_as_exists,\n -- superadmin fallback included: one acting outside their workspaces has no usr\n -- row but still authenticates.\n CASE WHEN on_behalf_of LIKE 'u/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $1::varchar\n AND u.username = substring(on_behalf_of from 3)\n UNION ALL\n SELECT 1 FROM password p WHERE p.super_admin\n AND (p.username = substring(on_behalf_of from 3)\n OR p.email = substring(on_behalf_of from 3))))\n WHEN on_behalf_of LIKE 'g/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM group_ g WHERE g.workspace_id = $1::varchar\n AND g.name = substring(on_behalf_of from 3)))\n ELSE\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $1::varchar\n AND u.username = on_behalf_of\n UNION ALL\n SELECT 1 FROM password p WHERE p.email = on_behalf_of\n AND p.super_admin))\n END, on_behalf_of_email, assets, modules\n FROM script\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "92723ff15a596e97edc71d24a5b653deb9d0e44259b22f2c6c5f43579f687360" +} diff --git a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json b/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json new file mode 100644 index 0000000000..da77dc1de5 --- /dev/null +++ b/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json @@ -0,0 +1,134 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "unique_id!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "resource_type!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "name!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "asset_path", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "materialized", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "materialize_strategy", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "tags!", + "type_info": "TextArray" + }, + { + "ordinal": 8, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "test_kind", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "test_column", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "test_args", + "type_info": "Jsonb" + }, + { + "ordinal": 12, + "name": "severity", + "type_info": "Text" + }, + { + "ordinal": 13, + "name": "attached_node", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "columns", + "type_info": "Jsonb" + }, + { + "ordinal": 15, + "name": "freshness", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "raw_code", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "original_file_path", + "type_info": "Text" + }, + { + "ordinal": 18, + "name": "script_visible!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + null + ] + }, + "hash": "9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12" +} diff --git a/backend/.sqlx/query-932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3.json b/backend/.sqlx/query-932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3.json new file mode 100644 index 0000000000..e2e3f374a6 --- /dev/null +++ b/backend/.sqlx/query-932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, on_behalf_of_email FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND deleted = false\n ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "on_behalf_of_email", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "932f7d0df4ab3e200b9eaea424f51756433c03af6e7810c2a5f9421515df8aa3" +} diff --git a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json new file mode 100644 index 0000000000..823a65cda3 --- /dev/null +++ b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47" +} diff --git a/backend/.sqlx/query-95c184a8f7d9396ed124bc1e2c3fddaeed002fe148f5d0434b41fda5d1203f27.json b/backend/.sqlx/query-95c184a8f7d9396ed124bc1e2c3fddaeed002fe148f5d0434b41fda5d1203f27.json new file mode 100644 index 0000000000..0fc9445931 --- /dev/null +++ b/backend/.sqlx/query-95c184a8f7d9396ed124bc1e2c3fddaeed002fe148f5d0434b41fda5d1203f27.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of, on_behalf_of_email)\n VALUES\n ('test-workspace', 'u/test-user/obo_member', 91001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user', 'test@windmill.dev'),\n ('test-workspace', 'u/test-user/obo_stranger', 91002, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2', 'test2@windmill.dev'),\n ('test-workspace', 'u/test-user/obo_group', 91003, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'g/all', 'group-all@windmill.dev'),\n ('test-workspace', 'u/test-user/obo_superadmin', 91004, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/ext-sa', 'sa@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "95c184a8f7d9396ed124bc1e2c3fddaeed002fe148f5d0434b41fda5d1203f27" +} diff --git a/backend/.sqlx/query-9621928b5736c56928368d17bd4111faeea6bfb54debbf8ee686db19364084fe.json b/backend/.sqlx/query-9621928b5736c56928368d17bd4111faeea6bfb54debbf8ee686db19364084fe.json new file mode 100644 index 0000000000..6c4bced8ce --- /dev/null +++ b/backend/.sqlx/query-9621928b5736c56928368d17bd4111faeea6bfb54debbf8ee686db19364084fe.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(\n SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2\n UNION ALL\n SELECT 1 FROM password WHERE email = $2 AND super_admin\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9621928b5736c56928368d17bd4111faeea6bfb54debbf8ee686db19364084fe" +} diff --git a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json b/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json new file mode 100644 index 0000000000..733478e0e9 --- /dev/null +++ b/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7" +} diff --git a/backend/.sqlx/query-96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9.json b/backend/.sqlx/query-96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9.json new file mode 100644 index 0000000000..5061c42c94 --- /dev/null +++ b/backend/.sqlx/query-96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM materialized_partition\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND asset_path = ANY($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "96ba9c55e1f3b6ec20ce4c21a72b12a4c8aa3dda05cceb694234107eb324cbf9" +} diff --git a/backend/.sqlx/query-972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0.json b/backend/.sqlx/query-972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0.json new file mode 100644 index 0000000000..ec34b5f46a --- /dev/null +++ b/backend/.sqlx/query-972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2\n AND NOT EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "972ee0dd58b98c1f0ff7242922e25b58b5ce504f2f4c4db952a796b3019a67d0" +} diff --git a/backend/.sqlx/query-9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294.json b/backend/.sqlx/query-9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294.json new file mode 100644 index 0000000000..42b8a2b09c --- /dev/null +++ b/backend/.sqlx/query-9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, tags, test_kind, test_column, test_args,\n attached_node)\n VALUES ($1, $2, $3, $4, 'test.p.accepted_values_orders_status', 'test',\n 'accepted_values_orders_status', '{}', 'accepted_values', 'status',\n '{\"values\": [\"gold\", \"silver\"]}'::jsonb, 'model.p.orders')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "9809fad5552d306f8cf04dfd9059be14ec5518c6ba8572362f5a849657237294" +} diff --git a/backend/.sqlx/query-98500dd427fdc64e711090387e08543ae8dfa457c6db2d74ebdc0cd748d72ac3.json b/backend/.sqlx/query-98500dd427fdc64e711090387e08543ae8dfa457c6db2d74ebdc0cd748d72ac3.json new file mode 100644 index 0000000000..46f6c9765e --- /dev/null +++ b/backend/.sqlx/query-98500dd427fdc64e711090387e08543ae8dfa457c6db2d74ebdc0cd748d72ac3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of'], to_jsonb($1::text))) WHERE typ IN ('script', 'flow') AND value->>'on_behalf_of' = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "98500dd427fdc64e711090387e08543ae8dfa457c6db2d74ebdc0cd748d72ac3" +} diff --git a/backend/.sqlx/query-9ac8b60c53631de2c3dab765ddc23cc99f9154e4239d228c0ef8f3726b0d19fc.json b/backend/.sqlx/query-9ac8b60c53631de2c3dab765ddc23cc99f9154e4239d228c0ef8f3726b0d19fc.json new file mode 100644 index 0000000000..777aa0feb2 --- /dev/null +++ b/backend/.sqlx/query-9ac8b60c53631de2c3dab765ddc23cc99f9154e4239d228c0ef8f3726b0d19fc.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT g.job_id, g.ingested_at FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.script_path = $2\n AND g.script_hash IS NOT DISTINCT FROM $3\n AND g.job_id = CASE WHEN EXISTS (\n SELECT 1 FROM dbt_graph_snapshot s\n WHERE s.workspace_id = $1 AND s.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "ingested_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9ac8b60c53631de2c3dab765ddc23cc99f9154e4239d228c0ef8f3726b0d19fc" +} diff --git a/backend/.sqlx/query-9be99f3104f595b0c7e73b67b120c2789ded855c69d5cd6e8b07c88710252ac5.json b/backend/.sqlx/query-9be99f3104f595b0c7e73b67b120c2789ded855c69d5cd6e8b07c88710252ac5.json new file mode 100644 index 0000000000..fe90532de7 --- /dev/null +++ b/backend/.sqlx/query-9be99f3104f595b0c7e73b67b120c2789ded855c69d5cd6e8b07c88710252ac5.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email)\n VALUES ('test-workspace', 'u/test-user/dg', 'script', '{\"on_behalf_of\": \"g/ops\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::json, 'test@windmill.dev'),\n ('test-workspace', 'u/test-user/du', 'script', '{\"on_behalf_of\": \"u/test-user-2\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::json, 'test@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "9be99f3104f595b0c7e73b67b120c2789ded855c69d5cd6e8b07c88710252ac5" +} diff --git a/backend/.sqlx/query-9cd98de055f34b86e98f6f1c060837644c813f45572896b42a55b4996cde1be8.json b/backend/.sqlx/query-9cd98de055f34b86e98f6f1c060837644c813f45572896b42a55b4996cde1be8.json new file mode 100644 index 0000000000..6aa872bfd4 --- /dev/null +++ b/backend/.sqlx/query-9cd98de055f34b86e98f6f1c060837644c813f45572896b42a55b4996cde1be8.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2 RETURNING timestamp, name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9cd98de055f34b86e98f6f1c060837644c813f45572896b42a55b4996cde1be8" +} diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f.json similarity index 88% rename from backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json rename to backend/.sqlx/query-9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f.json index 3bbd0f043e..b73ee34566 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n id,\n q.runnable_settings_handle,\n q.workspace_id,\n j.runnable_id as \"runnable_id: ScriptHash\",\n scheduled_for,\n parent_job,\n flow_innermost_root_job,\n runnable_path,\n kind as \"kind: JobKind\",\n started_at,\n permissioned_as,\n created_by,\n script_lang as \"script_lang: ScriptLang\",\n permissioned_as_email,\n flow_step_id,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n trigger,\n q.priority,\n concurrent_limit,\n q.tag,\n cache_ttl,\n cache_ignore_s3_path,\n r.ping as last_ping,\n worker,\n memory_peak,\n running\n FROM v2_job_queue q\n JOIN v2_job j USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1", + "query": "SELECT\n id,\n q.runnable_settings_handle,\n q.workspace_id,\n j.runnable_id as \"runnable_id: ScriptHash\",\n scheduled_for,\n parent_job,\n flow_innermost_root_job,\n runnable_path,\n kind as \"kind: JobKind\",\n started_at,\n permissioned_as,\n created_by,\n script_lang as \"script_lang: ScriptLang\",\n permissioned_as_email,\n flow_step_id,\n trigger_kind as \"trigger_kind: TriggerKindLabel\",\n trigger,\n q.priority,\n concurrent_limit,\n q.tag,\n cache_ttl,\n cache_ignore_s3_path,\n r.ping as last_ping,\n worker,\n memory_peak,\n running\n FROM v2_job_queue q\n JOIN v2_job j USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1", "describe": { "columns": [ { @@ -124,7 +124,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } @@ -142,7 +143,7 @@ }, { "ordinal": 15, - "name": "trigger_kind: JobTriggerKind", + "name": "trigger_kind: TriggerKindLabel", "type_info": { "Custom": { "name": "job_trigger_kind", @@ -259,5 +260,5 @@ false ] }, - "hash": "b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc" + "hash": "9d331b63c901071a7e82c0a4a6859c77a37ef784fdac78ca1e7a971e141e780f" } diff --git a/backend/.sqlx/query-9f4a95c35fe740d7f47db99335fe8a43b1df30076d6b487bdb6f762892bbec76.json b/backend/.sqlx/query-9f4a95c35fe740d7f47db99335fe8a43b1df30076d6b487bdb6f762892bbec76.json new file mode 100644 index 0000000000..721eeee984 --- /dev/null +++ b/backend/.sqlx/query-9f4a95c35fe740d7f47db99335fe8a43b1df30076d6b487bdb6f762892bbec76.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET on_behalf_of_email = $1 WHERE on_behalf_of_email = $2 AND (on_behalf_of IS NULL OR on_behalf_of NOT LIKE 'g/%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9f4a95c35fe740d7f47db99335fe8a43b1df30076d6b487bdb6f762892bbec76" +} diff --git a/backend/.sqlx/query-a0ef88bd0ec4823e4c8a226a790cf572fbf901326e80806bb6dcaa953f7b7b60.json b/backend/.sqlx/query-a0ef88bd0ec4823e4c8a226a790cf572fbf901326e80806bb6dcaa953f7b7b60.json new file mode 100644 index 0000000000..6097dab23e --- /dev/null +++ b/backend/.sqlx/query-a0ef88bd0ec4823e4c8a226a790cf572fbf901326e80806bb6dcaa953f7b7b60.json @@ -0,0 +1,53 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT service_config, webhook_token_hash, script_path, is_flow\n FROM native_trigger\n WHERE workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "service_config", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "webhook_token_hash", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + true, + false, + false, + false + ] + }, + "hash": "a0ef88bd0ec4823e4c8a226a790cf572fbf901326e80806bb6dcaa953f7b7b60" +} diff --git a/backend/.sqlx/query-a2b8356365a2e5e0efa5f851b2aa2d74a2f14a57ce8292f44b8fb482c7a07e2e.json b/backend/.sqlx/query-a2b8356365a2e5e0efa5f851b2aa2d74a2f14a57ce8292f44b8fb482c7a07e2e.json new file mode 100644 index 0000000000..b69ccb6cde --- /dev/null +++ b/backend/.sqlx/query-a2b8356365a2e5e0efa5f851b2aa2d74a2f14a57ce8292f44b8fb482c7a07e2e.json @@ -0,0 +1,78 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n flow_version.id AS version,\n flow_version.value->>'early_return' as early_return,\n flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor,\n flow_version.value->>'failure_module' IS NOT NULL as has_failure_module,\n (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled,\n flow.tag,\n flow.dedicated_worker,\n flow.on_behalf_of,\n flow.edited_by,\n flow.labels\n FROM\n flow_version\n INNER JOIN flow\n ON flow.path = flow_version.path AND\n flow.workspace_id = flow_version.workspace_id\n WHERE\n flow_version.workspace_id = $1 AND\n flow_version.path = $2 AND\n flow_version.id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "early_return", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "has_failure_module", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "chat_input_enabled", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "labels", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + null, + null, + null, + null, + true, + true, + true, + false, + true + ] + }, + "hash": "a2b8356365a2e5e0efa5f851b2aa2d74a2f14a57ce8292f44b8fb482c7a07e2e" +} diff --git a/backend/.sqlx/query-a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41.json b/backend/.sqlx/query-a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41.json new file mode 100644 index 0000000000..de91a548ad --- /dev/null +++ b/backend/.sqlx/query-a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT envs FROM script WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "envs", + "type_info": "VarcharArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + true + ] + }, + "hash": "a4651f0ec5daaa0cbf09a71a7da9e16ea112e0a3855cc730d2fe99259c64eb41" +} diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index 4824bba526..2b9ac8283b 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -134,7 +134,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-a606d8fc5876ef1e40278120d841994b8ec76d2008bcf749fcdcc607368bcdc3.json b/backend/.sqlx/query-a606d8fc5876ef1e40278120d841994b8ec76d2008bcf749fcdcc607368bcdc3.json new file mode 100644 index 0000000000..998563293a --- /dev/null +++ b/backend/.sqlx/query-a606d8fc5876ef1e40278120d841994b8ec76d2008bcf749fcdcc607368bcdc3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at)\n SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a606d8fc5876ef1e40278120d841994b8ec76d2008bcf749fcdcc607368bcdc3" +} diff --git a/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json b/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json new file mode 100644 index 0000000000..0532a5d3a0 --- /dev/null +++ b/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET\n ai_config = source_ws.ai_config,\n large_file_storage = source_ws.large_file_storage,\n ducklake = source_ws.ducklake,\n dbt_warehouses = source_ws.dbt_warehouses,\n datatable = source_ws.datatable,\n git_app_installations = source_ws.git_app_installations\n FROM workspace_settings source_ws\n WHERE source_ws.workspace_id = $1\n AND workspace_settings.workspace_id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a" +} diff --git a/backend/.sqlx/query-a6f59e8751b79d23fcd2a8204fbc38002fc6d0cf7e4a83f21d31fb28bd98efb8.json b/backend/.sqlx/query-a6f59e8751b79d23fcd2a8204fbc38002fc6d0cf7e4a83f21d31fb28bd98efb8.json new file mode 100644 index 0000000000..e7a544525f --- /dev/null +++ b/backend/.sqlx/query-a6f59e8751b79d23fcd2a8204fbc38002fc6d0cf7e4a83f21d31fb28bd98efb8.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app(workspace_id, path, summary, policy, versions)\n VALUES ('test-workspace', 'u/test-user/g', '', '{\"on_behalf_of\": \"g/ops\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::jsonb, '{}'),\n ('test-workspace', 'u/test-user/u', '', '{\"on_behalf_of\": \"u/test-user-2\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::jsonb, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a6f59e8751b79d23fcd2a8204fbc38002fc6d0cf7e4a83f21d31fb28bd98efb8" +} diff --git a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json index 0d1bb6cb41..0faa9d0e33 100644 --- a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json +++ b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json @@ -39,7 +39,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-a8eb013bbbb4d8cdc0d4e5e3f46e805e039e57ef9f7be07d2e738d6628e585d5.json b/backend/.sqlx/query-a8eb013bbbb4d8cdc0d4e5e3f46e805e039e57ef9f7be07d2e738d6628e585d5.json new file mode 100644 index 0000000000..4419bf7a6d --- /dev/null +++ b/backend/.sqlx/query-a8eb013bbbb4d8cdc0d4e5e3f46e805e039e57ef9f7be07d2e738d6628e585d5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of'], to_jsonb($1::text)), ARRAY['on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('script', 'flow') AND value->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a8eb013bbbb4d8cdc0d4e5e3f46e805e039e57ef9f7be07d2e738d6628e585d5" +} diff --git a/backend/.sqlx/query-aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c.json b/backend/.sqlx/query-aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c.json new file mode 100644 index 0000000000..e417d1e948 --- /dev/null +++ b/backend/.sqlx/query-aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms)\n VALUES ($1, 'private', 'private', '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "aa463baab01d9530ae97ace8b5a296ce9855bce50ea8c5918941564a696c029c" +} diff --git a/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json b/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json index 04131f8f88..0bba0710e8 100644 --- a/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json +++ b/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-ab46ac8e14a05a79340511ad081f9f540139e1f593ac6f80221378ff856954cb.json b/backend/.sqlx/query-ab46ac8e14a05a79340511ad081f9f540139e1f593ac6f80221378ff856954cb.json new file mode 100644 index 0000000000..6ed6813681 --- /dev/null +++ b/backend/.sqlx/query-ab46ac8e14a05a79340511ad081f9f540139e1f593ac6f80221378ff856954cb.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels)\n SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels\n FROM flow\n WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ab46ac8e14a05a79340511ad081f9f540139e1f593ac6f80221378ff856954cb" +} diff --git a/backend/.sqlx/query-ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9.json b/backend/.sqlx/query-ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9.json new file mode 100644 index 0000000000..52ebcb5422 --- /dev/null +++ b/backend/.sqlx/query-ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT hash FROM script WHERE workspace_id = $1 AND path = $2 AND deleted = false AND archived = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ac113ee13ac2b6e8da5ebb6bf9f8207e1b46ba34fcb0f6957caef4be49bbbaa9" +} diff --git a/backend/.sqlx/query-af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7.json b/backend/.sqlx/query-af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7.json new file mode 100644 index 0000000000..f8129440a8 --- /dev/null +++ b/backend/.sqlx/query-af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_state SET workspace_id = $1\n WHERE workspace_id = $2 AND job_id IN (SELECT id FROM v2_job WHERE workspace_id = $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "af2f8c70ad5a14d35048211631b250879d18751a53597ef262b63d1e29bcb6a7" +} diff --git a/backend/.sqlx/query-b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39.json b/backend/.sqlx/query-b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39.json new file mode 100644 index 0000000000..10adc6e68d --- /dev/null +++ b/backend/.sqlx/query-b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_edge WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b19f5156b3e36b5e831ce792637bf54a3f182c03cef0aef06a1c75b45992cd39" +} diff --git a/backend/.sqlx/query-b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0.json b/backend/.sqlx/query-b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0.json index 1f21f33ce2..806b923acb 100644 --- a/backend/.sqlx/query-b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0.json +++ b/backend/.sqlx/query-b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0.json @@ -154,7 +154,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506.json b/backend/.sqlx/query-b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506.json new file mode 100644 index 0000000000..e29c1c0dbb --- /dev/null +++ b/backend/.sqlx/query-b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "b24c1ed9aa3b856eed3da73eadb99a11874d6680f584324827f0ae59bb9fe506" +} diff --git a/backend/.sqlx/query-b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4.json b/backend/.sqlx/query-b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4.json new file mode 100644 index 0000000000..b865865c1d --- /dev/null +++ b/backend/.sqlx/query-b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b2dd773eebe0005eb4220f19e6a01aa9d12e461a89243bd0b651d1e5b837b6f4" +} diff --git a/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json b/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json index eab78c7de8..9187cc29b7 100644 --- a/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json +++ b/backend/.sqlx/query-b44afcf1b9c047ac525638f0952c2cb01d65b1b46693331ac157dfca0dab6824.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c.json b/backend/.sqlx/query-b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c.json new file mode 100644 index 0000000000..3aaa58da40 --- /dev/null +++ b/backend/.sqlx/query-b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_state SET script_path = $3 WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "b546a7a68143a3b0de1d900e5f635b8a1e4d1ba18298ace1602defacb9e67f0c" +} diff --git a/backend/.sqlx/query-b6303603bebdcf6d5bad2006ce65fe7900e33a843711934799b7259d1c451da6.json b/backend/.sqlx/query-b6303603bebdcf6d5bad2006ce65fe7900e33a843711934799b7259d1c451da6.json new file mode 100644 index 0000000000..218cf8252f --- /dev/null +++ b/backend/.sqlx/query-b6303603bebdcf6d5bad2006ce65fe7900e33a843711934799b7259d1c451da6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE script SET on_behalf_of = $1 WHERE on_behalf_of = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b6303603bebdcf6d5bad2006ce65fe7900e33a843711934799b7259d1c451da6" +} diff --git a/backend/.sqlx/query-b9686c115e83ad0725503e2c076960914f825eaa2b44281a4e147cea6e886e7a.json b/backend/.sqlx/query-b9686c115e83ad0725503e2c076960914f825eaa2b44281a4e147cea6e886e7a.json new file mode 100644 index 0000000000..ebb5cee72c --- /dev/null +++ b/backend/.sqlx/query-b9686c115e83ad0725503e2c076960914f825eaa2b44281a4e147cea6e886e7a.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM native_trigger WHERE workspace_id = $1 AND service_name = $2 RETURNING webhook_token_hash", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "webhook_token_hash", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + } + ] + }, + "nullable": [ + false + ] + }, + "hash": "b9686c115e83ad0725503e2c076960914f825eaa2b44281a4e147cea6e886e7a" +} diff --git a/backend/.sqlx/query-bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472.json b/backend/.sqlx/query-bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472.json new file mode 100644 index 0000000000..09367d07d3 --- /dev/null +++ b/backend/.sqlx/query-bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_progress SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bc59b7d863fc9df10cddf7487d6c2a3bd3ca8334f6cb5eaf42e14265a32ae472" +} diff --git a/backend/.sqlx/query-bc75df084b33779868317fe5c1d09bc8f1e01228aeb60fb1cdc034b4422888e0.json b/backend/.sqlx/query-bc75df084b33779868317fe5c1d09bc8f1e01228aeb60fb1cdc034b4422888e0.json new file mode 100644 index 0000000000..a245f1868f --- /dev/null +++ b/backend/.sqlx/query-bc75df084b33779868317fe5c1d09bc8f1e01228aeb60fb1cdc034b4422888e0.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow SET on_behalf_of_email = $1 WHERE on_behalf_of_email = $2 AND (on_behalf_of IS NULL OR on_behalf_of NOT LIKE 'g/%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bc75df084b33779868317fe5c1d09bc8f1e01228aeb60fb1cdc034b4422888e0" +} diff --git a/backend/.sqlx/query-bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167.json b/backend/.sqlx/query-bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167.json new file mode 100644 index 0000000000..501a8460a1 --- /dev/null +++ b/backend/.sqlx/query-bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bd293b1e468f70dcc21ab4a91eb5225058fb14d29a8a4de04fe161af0ff2d167" +} diff --git a/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json b/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json index 2e194c9580..be50a16b17 100644 --- a/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json +++ b/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json @@ -41,7 +41,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index 809deb1a55..32e4c5404e 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -78,7 +78,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-bea1144ea4d50aff717654eb1c2fd3a98c72172fd907c5a710ae50abbd02fd92.json b/backend/.sqlx/query-bea1144ea4d50aff717654eb1c2fd3a98c72172fd907c5a710ae50abbd02fd92.json new file mode 100644 index 0000000000..ea3dbea9fb --- /dev/null +++ b/backend/.sqlx/query-bea1144ea4d50aff717654eb1c2fd3a98c72172fd907c5a710ae50abbd02fd92.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO group_(workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "bea1144ea4d50aff717654eb1c2fd3a98c72172fd907c5a710ae50abbd02fd92" +} diff --git a/backend/.sqlx/query-bf45139e3ae5c5c16591afb26c51c01c1e2fd56a2dc2dbd1ed41f08be7e61e08.json b/backend/.sqlx/query-bf45139e3ae5c5c16591afb26c51c01c1e2fd56a2dc2dbd1ed41f08be7e61e08.json new file mode 100644 index 0000000000..d5763deb84 --- /dev/null +++ b/backend/.sqlx/query-bf45139e3ae5c5c16591afb26c51c01c1e2fd56a2dc2dbd1ed41f08be7e61e08.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, value->>'on_behalf_of_email' AS email, value->>'on_behalf_of' AS principal FROM draft WHERE workspace_id = 'test-workspace' ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "principal", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "bf45139e3ae5c5c16591afb26c51c01c1e2fd56a2dc2dbd1ed41f08be7e61e08" +} diff --git a/backend/.sqlx/query-bf63f1e3c2f06b1133a4cd6ced5a3f7752e79027a2a8636808bb41628b789ebc.json b/backend/.sqlx/query-bf63f1e3c2f06b1133a4cd6ced5a3f7752e79027a2a8636808bb41628b789ebc.json new file mode 100644 index 0000000000..a0ab56888b --- /dev/null +++ b/backend/.sqlx/query-bf63f1e3c2f06b1133a4cd6ced5a3f7752e79027a2a8636808bb41628b789ebc.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE workspace_id = 'test-workspace' AND username = 'test-user-2'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "bf63f1e3c2f06b1133a4cd6ced5a3f7752e79027a2a8636808bb41628b789ebc" +} diff --git a/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json b/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json index a051b068a5..df982f3296 100644 --- a/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json +++ b/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json @@ -34,7 +34,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-c13f203478d6bc623c27bba9b6fc6e9c9d36bcf9f7138422414f31f0319e6cfb.json b/backend/.sqlx/query-c13f203478d6bc623c27bba9b6fc6e9c9d36bcf9f7138422414f31f0319e6cfb.json new file mode 100644 index 0000000000..a1bf434814 --- /dev/null +++ b/backend/.sqlx/query-c13f203478d6bc623c27bba9b6fc6e9c9d36bcf9f7138422414f31f0319e6cfb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false AND deleted = false)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c13f203478d6bc623c27bba9b6fc6e9c9d36bcf9f7138422414f31f0319e6cfb" +} diff --git a/backend/.sqlx/query-c159975f7b9ad9c8eeef340ca03ed4919122d653e1235d0847bfad84f0a3e539.json b/backend/.sqlx/query-c159975f7b9ad9c8eeef340ca03ed4919122d653e1235d0847bfad84f0a3e539.json new file mode 100644 index 0000000000..4be222b94d --- /dev/null +++ b/backend/.sqlx/query-c159975f7b9ad9c8eeef340ca03ed4919122d653e1235d0847bfad84f0a3e539.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, policy->>'on_behalf_of_email' AS email FROM app WHERE workspace_id = 'test-workspace' ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "c159975f7b9ad9c8eeef340ca03ed4919122d653e1235d0847bfad84f0a3e539" +} diff --git a/backend/.sqlx/query-c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3.json b/backend/.sqlx/query-c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3.json new file mode 100644 index 0000000000..5fb1a353e7 --- /dev/null +++ b/backend/.sqlx/query-c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_edge\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "c1a85cf2d1555ae33bbdaf85cf243d44a23f2fe2abf7a9ff8f271f9dfb1433e3" +} diff --git a/backend/.sqlx/query-c1fb844ac8fb9293510d08dfb1cec03751a661076ee3250446b5157c43defc97.json b/backend/.sqlx/query-c1fb844ac8fb9293510d08dfb1cec03751a661076ee3250446b5157c43defc97.json new file mode 100644 index 0000000000..cd2dead06d --- /dev/null +++ b/backend/.sqlx/query-c1fb844ac8fb9293510d08dfb1cec03751a661076ee3250446b5157c43defc97.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot\n (workspace_id, script_path, script_hash, job_id, digest, ingested_at)\n VALUES ($1, $2, $3, $4, $5, now())\n ON CONFLICT (workspace_id, script_path, script_hash, job_id)\n WHERE script_hash IS NOT NULL\n DO UPDATE SET digest = EXCLUDED.digest, ingested_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c1fb844ac8fb9293510d08dfb1cec03751a661076ee3250446b5157c43defc97" +} diff --git a/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json index 7981dbc983..9844615122 100644 --- a/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json +++ b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json @@ -16,7 +16,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } @@ -87,7 +88,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241.json b/backend/.sqlx/query-c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241.json new file mode 100644 index 0000000000..d8f9fbcee4 --- /dev/null +++ b/backend/.sqlx/query-c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_run_state (workspace_id, script_path, permissioned_as, identity, args, run_results, job_id, retryable, updated_at)\n SELECT $1::varchar, $2::varchar, $7::varchar, $3::text, $4::jsonb, $5::text, $6::uuid, $8::boolean, now()\n WHERE EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false\n AND language = 'dbt')\n ON CONFLICT (workspace_id, script_path, permissioned_as) DO UPDATE SET\n identity = EXCLUDED.identity, args = EXCLUDED.args,\n run_results = EXCLUDED.run_results, job_id = EXCLUDED.job_id,\n retryable = EXCLUDED.retryable, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Jsonb", + "Text", + "Uuid", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "c2e5f233e134ccc70b05d62239aaf736b456dbdd7d358e58744aa565e5889241" +} diff --git a/backend/.sqlx/query-c6706971489603027edc678f9d6c42e8341d5d3c5e4f46a53b8dc637484dd760.json b/backend/.sqlx/query-c6706971489603027edc678f9d6c42e8341d5d3c5e4f46a53b8dc637484dd760.json new file mode 100644 index 0000000000..27087b4c4d --- /dev/null +++ b/backend/.sqlx/query-c6706971489603027edc678f9d6c42e8341d5d3c5e4f46a53b8dc637484dd760.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM script WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived AND NOT deleted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c6706971489603027edc678f9d6c42e8341d5d3c5e4f46a53b8dc637484dd760" +} diff --git a/backend/.sqlx/query-c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558.json b/backend/.sqlx/query-c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558.json new file mode 100644 index 0000000000..8bc10809dc --- /dev/null +++ b/backend/.sqlx/query-c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT deleted FROM script WHERE workspace_id = $1 AND hash = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "deleted", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c868e87f7ccc5ce19c421c857f67285177bd4eaa3a14af5bf08dbc32c7bda558" +} diff --git a/backend/.sqlx/query-c8cdc44f76411a3a01b02eed6647e8319c368ca9ccdc095f94945cda082f8857.json b/backend/.sqlx/query-c8cdc44f76411a3a01b02eed6647e8319c368ca9ccdc095f94945cda082f8857.json new file mode 100644 index 0000000000..ac8ae1fa0b --- /dev/null +++ b/backend/.sqlx/query-c8cdc44f76411a3a01b02eed6647e8319c368ca9ccdc095f94945cda082f8857.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, permissioned_as, permissioned_as_email FROM v2_job WHERE id = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "permissioned_as_email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "c8cdc44f76411a3a01b02eed6647e8319c368ca9ccdc095f94945cda082f8857" +} diff --git a/backend/.sqlx/query-cbc68952e976e3daeba2dba398de11f81ef4d93f6f6d164269a5190af85512d9.json b/backend/.sqlx/query-cbc68952e976e3daeba2dba398de11f81ef4d93f6f6d164269a5190af85512d9.json new file mode 100644 index 0000000000..9ea2efa7ac --- /dev/null +++ b/backend/.sqlx/query-cbc68952e976e3daeba2dba398de11f81ef4d93f6f6d164269a5190af85512d9.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of FROM script WHERE path = 'u/test-user/s' AND workspace_id = 'test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "cbc68952e976e3daeba2dba398de11f81ef4d93f6f6d164269a5190af85512d9" +} diff --git a/backend/.sqlx/query-d01d85e7e5371591f0f2e5843448f464ed38bd4c314a19cc9a9e6f7d89b4a173.json b/backend/.sqlx/query-d01d85e7e5371591f0f2e5843448f464ed38bd4c314a19cc9a9e6f7d89b4a173.json new file mode 100644 index 0000000000..98550a98e9 --- /dev/null +++ b/backend/.sqlx/query-d01d85e7e5371591f0f2e5843448f464ed38bd4c314a19cc9a9e6f7d89b4a173.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of_email FROM script WHERE path = 'u/test-user/obo_recorded' AND workspace_id = 'test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of_email", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "d01d85e7e5371591f0f2e5843448f464ed38bd4c314a19cc9a9e6f7d89b4a173" +} diff --git a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json b/backend/.sqlx/query-d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b.json similarity index 94% rename from backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json rename to backend/.sqlx/query-d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b.json index 0bd066afd1..0da783f056 100644 --- a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json +++ b/backend/.sqlx/query-d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n v2_job_queue.runnable_settings_handle,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n cache_ignore_s3_path,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n visible_to_owner,\n NULL as permissioned_as_end_user_email\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", + "query": "SELECT\n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n v2_job_queue.runnable_settings_handle,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n cache_ignore_s3_path,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: TriggerKindLabel\",\n visible_to_owner,\n NULL as permissioned_as_end_user_email\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", "describe": { "columns": [ { @@ -149,7 +149,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } @@ -227,7 +228,7 @@ }, { "ordinal": 32, - "name": "trigger_kind: JobTriggerKind", + "name": "trigger_kind: TriggerKindLabel", "type_info": { "Custom": { "name": "job_trigger_kind", @@ -313,5 +314,5 @@ null ] }, - "hash": "d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9" + "hash": "d0df9d1fda20f0366505bf6abf67446a5e8e23d87eebc46bbb90bdbdf128935b" } diff --git a/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json b/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json index c4526daccc..64b4056596 100644 --- a/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json +++ b/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json @@ -69,7 +69,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index 2660ca05ef..cd1e0c4c39 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -78,7 +78,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b.json b/backend/.sqlx/query-d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b.json new file mode 100644 index 0000000000..edd7f2f97a --- /dev/null +++ b/backend/.sqlx/query-d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_graph_snapshot\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "d735f355d9eb2612b563b606c6c897c5ad764a882b2c9054a7665dd68b7a528b" +} diff --git a/backend/.sqlx/query-d75bbac607ef4439d8aef2b33d1b2a035d2ecb8169fd26e6a1cb2b49ead6c4ea.json b/backend/.sqlx/query-d75bbac607ef4439d8aef2b33d1b2a035d2ecb8169fd26e6a1cb2b49ead6c4ea.json new file mode 100644 index 0000000000..02845d173a --- /dev/null +++ b/backend/.sqlx/query-d75bbac607ef4439d8aef2b33d1b2a035d2ecb8169fd26e6a1cb2b49ead6c4ea.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name)\n VALUES ('ext@windmill.dev', 'not-a-real-hash', 'password', true, true, 'Ext')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d75bbac607ef4439d8aef2b33d1b2a035d2ecb8169fd26e6a1cb2b49ead6c4ea" +} diff --git a/backend/.sqlx/query-da10117644334b74bdb68e3f9d7ef37cd4aa442e68f80d8ce5f2b74a20d5146f.json b/backend/.sqlx/query-da10117644334b74bdb68e3f9d7ef37cd4aa442e68f80d8ce5f2b74a20d5146f.json new file mode 100644 index 0000000000..55b950ab4f --- /dev/null +++ b/backend/.sqlx/query-da10117644334b74bdb68e3f9d7ef37cd4aa442e68f80d8ce5f2b74a20d5146f.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of, on_behalf_of_email FROM script WHERE path = 'f/shared/obo' AND workspace_id = 'test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "on_behalf_of_email", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + true + ] + }, + "hash": "da10117644334b74bdb68e3f9d7ef37cd4aa442e68f80d8ce5f2b74a20d5146f" +} diff --git a/backend/.sqlx/query-dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6.json b/backend/.sqlx/query-dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6.json new file mode 100644 index 0000000000..520d04c762 --- /dev/null +++ b/backend/.sqlx/query-dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_edge WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "dbe64a8367fad74866b15f96565ef1ae7c58848714974a8c803085a0c0e655d6" +} diff --git a/backend/.sqlx/query-dcbf4630616d214c510dd8bda9cb77215453c094cc1caab16ae4b4e5148b0c58.json b/backend/.sqlx/query-dcbf4630616d214c510dd8bda9cb77215453c094cc1caab16ae4b4e5148b0c58.json new file mode 100644 index 0000000000..363c96f694 --- /dev/null +++ b/backend/.sqlx/query-dcbf4630616d214c510dd8bda9cb77215453c094cc1caab16ae4b4e5148b0c58.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs)\n SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs\n FROM flow WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dcbf4630616d214c510dd8bda9cb77215453c094cc1caab16ae4b4e5148b0c58" +} diff --git a/backend/.sqlx/query-e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165.json b/backend/.sqlx/query-e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165.json new file mode 100644 index 0000000000..36161cb99c --- /dev/null +++ b/backend/.sqlx/query-e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_node\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "e013b906ba3e95350ebcda6a8b8cc3ee04f6782587bc34d44338d2b2cad03165" +} diff --git a/backend/.sqlx/query-e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609.json b/backend/.sqlx/query-e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609.json new file mode 100644 index 0000000000..4967ac0212 --- /dev/null +++ b/backend/.sqlx/query-e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_run_progress\n SET status = 'failed',\n error = COALESCE(error, 'the run ended before this model finished')\n WHERE workspace_id = $1 AND job_id = $2 AND status = 'running'\n AND NOT (asset_path = ANY($3))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e01e9040f2336d9db41638842c5a4a07af79130e1dbfd91f1601c04721186609" +} diff --git a/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json b/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json index 037c796fc7..0470656ba6 100644 --- a/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json +++ b/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json @@ -17,7 +17,8 @@ "variable", "ducklake", "datatable", - "volume" + "volume", + "dbt" ] } } diff --git a/backend/.sqlx/query-e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7.json b/backend/.sqlx/query-e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7.json new file mode 100644 index 0000000000..9509a76604 --- /dev/null +++ b/backend/.sqlx/query-e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script_trigger\n WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2\n AND trigger_kind = 'asset' AND trigger_ref LIKE 'dbt://%'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e3fd8cfde2ae3e374b7c4ffcab91fcce10865992242cd053779760cc68909ff7" +} diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index ad34773e36..29d3eede53 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -134,7 +134,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-e782878f409796f1fe2a773f054f0b3f63185a6e6a9f4e0dfe961dd466a41808.json b/backend/.sqlx/query-e782878f409796f1fe2a773f054f0b3f63185a6e6a9f4e0dfe961dd466a41808.json new file mode 100644 index 0000000000..73f5c2e9f7 --- /dev/null +++ b/backend/.sqlx/query-e782878f409796f1fe2a773f054f0b3f63185a6e6a9f4e0dfe961dd466a41808.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH inserted AS (\n INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs)\n SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM inserted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e782878f409796f1fe2a773f054f0b3f63185a6e6a9f4e0dfe961dd466a41808" +} diff --git a/backend/.sqlx/query-e7f617f9abc523708567ee415f95150048be5d7d0e494fbd3aaf550f4c4d8125.json b/backend/.sqlx/query-e7f617f9abc523708567ee415f95150048be5d7d0e494fbd3aaf550f4c4d8125.json new file mode 100644 index 0000000000..a96671ab04 --- /dev/null +++ b/backend/.sqlx/query-e7f617f9abc523708567ee415f95150048be5d7d0e494fbd3aaf550f4c4d8125.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}') ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e7f617f9abc523708567ee415f95150048be5d7d0e494fbd3aaf550f4c4d8125" +} diff --git a/backend/.sqlx/query-ad25201d0eea65972234cade87a95d8cd99fc26e5bd466942423cbd09efcebe4.json b/backend/.sqlx/query-eaad94dd1b419f814af4cb54f2843e8d12a191ca97954c499ae5dc022410a240.json similarity index 74% rename from backend/.sqlx/query-ad25201d0eea65972234cade87a95d8cd99fc26e5bd466942423cbd09efcebe4.json rename to backend/.sqlx/query-eaad94dd1b419f814af4cb54f2843e8d12a191ca97954c499ae5dc022410a240.json index 7b989b68f5..29aa952292 100644 --- a/backend/.sqlx/query-ad25201d0eea65972234cade87a95d8cd99fc26e5bd466942423cbd09efcebe4.json +++ b/backend/.sqlx/query-eaad94dd1b419f814af4cb54f2843e8d12a191ca97954c499ae5dc022410a240.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO azure_trigger (\n azure_resource_path, azure_mode, scope_resource_id, topic_name,\n subscription_name, event_type_filters,\n push_auth_config, workspace_id, path, script_path, is_flow,\n permissioned_as, mode, edited_by, email,\n error_handler_path, error_handler_args, retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7,\n $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18\n )\n ", + "query": "\n INSERT INTO azure_trigger (\n azure_resource_path, azure_mode, scope_resource_id, topic_name,\n subscription_name, event_type_filters,\n push_auth_config, workspace_id, path, script_path, is_flow,\n permissioned_as, mode, edited_by,\n error_handler_path, error_handler_args, retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7,\n $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ", "describe": { "columns": [], "parameters": { @@ -42,12 +42,11 @@ }, "Varchar", "Varchar", - "Varchar", "Jsonb", "Jsonb" ] }, "nullable": [] }, - "hash": "ad25201d0eea65972234cade87a95d8cd99fc26e5bd466942423cbd09efcebe4" + "hash": "eaad94dd1b419f814af4cb54f2843e8d12a191ca97954c499ae5dc022410a240" } diff --git a/backend/.sqlx/query-ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa.json b/backend/.sqlx/query-ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa.json new file mode 100644 index 0000000000..d6e1a1bb57 --- /dev/null +++ b/backend/.sqlx/query-ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_run_progress\n (workspace_id, job_id, asset_kind, asset_path, status, row_count, error, updated_at)\n VALUES ($1, $2, 'dbt', $3, $4, $5, $6, now())\n ON CONFLICT (workspace_id, job_id, asset_kind, asset_path)\n DO UPDATE SET status = EXCLUDED.status, row_count = EXCLUDED.row_count,\n error = EXCLUDED.error, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "Varchar", + { + "Custom": { + "name": "materialization_status", + "kind": { + "Enum": [ + "running", + "materialized", + "failed" + ] + } + } + }, + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ec2ed5203effe26c9584b266c5ccbc2a399e3a0e126c9ba7d0558d37c16926fa" +} diff --git a/backend/.sqlx/query-ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9.json b/backend/.sqlx/query-ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9.json new file mode 100644 index 0000000000..90e4ba1e36 --- /dev/null +++ b/backend/.sqlx/query-ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job_queue q JOIN v2_job j ON j.id = q.id\n WHERE q.id = $1 AND j.workspace_id = $2 AND q.tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ed421d15d2f9883196619418b3ebd7c687900b66cd8c3a173f2e3410e4ed86d9" +} diff --git a/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json b/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json index 33b0b1f144..2357865ebc 100644 --- a/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json +++ b/backend/.sqlx/query-edd6c09b7f012588788fd3c572d20eb439a80d52ae75ebd25128ffce759cd313.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-ee85a02a8e3abe3385524c9e549a9f1685387f527eb8739125f6a75ecfb991ac.json b/backend/.sqlx/query-ee85a02a8e3abe3385524c9e549a9f1685387f527eb8739125f6a75ecfb991ac.json new file mode 100644 index 0000000000..fe4797b43b --- /dev/null +++ b/backend/.sqlx/query-ee85a02a8e3abe3385524c9e549a9f1685387f527eb8739125f6a75ecfb991ac.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n flow\n SET\n path = $1,\n summary = $2,\n description = $3,\n dependency_job = NULL,\n lock_error_logs = '',\n tag = $4,\n dedicated_worker = $5,\n visible_to_runner_only = $6,\n ws_error_handler_muted = $7,\n value = $8,\n schema = $9::text::json,\n edited_by = $10,\n edited_at = now(),\n labels = COALESCE($13, labels),\n on_behalf_of = $14,\n on_behalf_of_email = $15\n WHERE\n path = $11 AND workspace_id = $12", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Varchar", + "Bool", + "Bool", + "Bool", + "Jsonb", + "Text", + "Varchar", + "Text", + "Text", + "TextArray", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ee85a02a8e3abe3385524c9e549a9f1685387f527eb8739125f6a75ecfb991ac" +} diff --git a/backend/.sqlx/query-eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c.json b/backend/.sqlx/query-eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c.json new file mode 100644 index 0000000000..63babb575e --- /dev/null +++ b/backend/.sqlx/query-eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id,\n digest, relation_root_at_last_ingest, ingested_at)\n SELECT $2, script_path, script_hash, job_id, digest, relation_root_at_last_ingest,\n ingested_at\n FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "eed8122fdcd007543dc6ca28b216845c6249c4598c4e16aa50067fdc6136451c" +} diff --git a/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json b/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json index ad6028c14f..1bd565d00c 100644 --- a/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json +++ b/backend/.sqlx/query-f0858450021df721d8a48b1b5dc887c5424562acd9769c80c5899193ef16b56b.json @@ -44,7 +44,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce.json b/backend/.sqlx/query-f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e.json similarity index 83% rename from backend/.sqlx/query-88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce.json rename to backend/.sqlx/query-f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e.json index 9097eb8584..81034a3861 100644 --- a/backend/.sqlx/query-88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce.json +++ b/backend/.sqlx/query-f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n ai_config,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n ai_config,\n dbt_warehouses,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", "describe": { "columns": [ { @@ -80,81 +80,86 @@ }, { "ordinal": 15, - "name": "large_file_storage", + "name": "dbt_warehouses", "type_info": "Jsonb" }, { "ordinal": 16, - "name": "datatable", + "name": "large_file_storage", "type_info": "Jsonb" }, { "ordinal": 17, - "name": "ducklake", + "name": "datatable", "type_info": "Jsonb" }, { "ordinal": 18, - "name": "git_sync", + "name": "ducklake", "type_info": "Jsonb" }, { "ordinal": 19, - "name": "deploy_ui", + "name": "git_sync", "type_info": "Jsonb" }, { "ordinal": 20, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 21, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 21, + "ordinal": 22, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 22, + "ordinal": 23, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 23, + "ordinal": 24, "name": "color", "type_info": "Varchar" }, { - "ordinal": 24, + "ordinal": 25, "name": "operator_settings", "type_info": "Jsonb" }, { - "ordinal": 25, + "ordinal": 26, "name": "git_app_installations", "type_info": "Jsonb" }, { - "ordinal": 26, + "ordinal": 27, "name": "auto_invite", "type_info": "Jsonb" }, { - "ordinal": 27, + "ordinal": 28, "name": "error_handler", "type_info": "Jsonb" }, { - "ordinal": 28, + "ordinal": 29, "name": "success_handler", "type_info": "Jsonb" }, { - "ordinal": 29, + "ordinal": 30, "name": "public_app_execution_limit_per_minute", "type_info": "Int4" }, { - "ordinal": 30, + "ordinal": 31, "name": "error_handler_fallback_to_instance_alerts", "type_info": "Bool" } @@ -190,6 +195,7 @@ true, true, true, + true, false, true, true, @@ -198,5 +204,5 @@ false ] }, - "hash": "88a6a86285fcfaa778e9716d5072f555dff175feb704c3d825d6100defe91bce" + "hash": "f1a34561f208c05d75b2f981cd55465e579dd71a9439a2182ef05d9d3cfe462e" } diff --git a/backend/.sqlx/query-f1b13ed0a8b834b0a55bfc7ef6470c2a3aaef44f19fbff2673d7bbd9d9a9aca2.json b/backend/.sqlx/query-f1b13ed0a8b834b0a55bfc7ef6470c2a3aaef44f19fbff2673d7bbd9d9a9aca2.json new file mode 100644 index 0000000000..43f0740aad --- /dev/null +++ b/backend/.sqlx/query-f1b13ed0a8b834b0a55bfc7ef6470c2a3aaef44f19fbff2673d7bbd9d9a9aca2.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, runnable_path,\n CASE WHEN kind = 'script' THEN runnable_id END AS script_hash,\n -- Whether this job PARSED a graph of its own with no version\n -- behind it: the dbt editor refreshing its buffer. That graph is\n -- reachable no other way, so the job pins to it; every other\n -- versionless job keeps answering with the workspace graph.\n EXISTS (SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $2 AND g.job_id = $1\n AND g.script_hash IS NULL) AS \"editor_graph!\"\n FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_hash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "editor_graph!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + true, + null, + null + ] + }, + "hash": "f1b13ed0a8b834b0a55bfc7ef6470c2a3aaef44f19fbff2673d7bbd9d9a9aca2" +} diff --git a/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json index 07191c20b8..197229bb4c 100644 --- a/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json +++ b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json @@ -41,7 +41,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json b/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json index 12eceb0110..57d80d6aaa 100644 --- a/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json +++ b/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json @@ -39,7 +39,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } @@ -86,7 +87,8 @@ "java", "duckdb", "ruby", - "rlang" + "rlang", + "dbt" ] } } diff --git a/backend/.sqlx/query-f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194.json b/backend/.sqlx/query-f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194.json new file mode 100644 index 0000000000..8cae250396 --- /dev/null +++ b/backend/.sqlx/query-f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "f45962e947ae236266f8eb9b08ca6420ac7e0d548fb59473a2f7ac5be2ee5194" +} diff --git a/backend/.sqlx/query-f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8.json b/backend/.sqlx/query-f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8.json new file mode 100644 index 0000000000..0c2078f793 --- /dev/null +++ b/backend/.sqlx/query-f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT asset_kind AS \"asset_kind: windmill_common::assets::AssetKind\", asset_path,\n status::text AS \"status!\", row_count, error\n FROM materialized_partition\n WHERE workspace_id = $1 AND job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume", + "dbt" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "row_count", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + false, + false, + null, + true, + true + ] + }, + "hash": "f5eedc14b082c88a1096e61279faa73793c99a2ab5793bebc5924146f60d19a8" +} diff --git a/backend/.sqlx/query-f60339bdf63e72c63df45530be4e13380720237a1a310d8acf5e975189981422.json b/backend/.sqlx/query-f60339bdf63e72c63df45530be4e13380720237a1a310d8acf5e975189981422.json new file mode 100644 index 0000000000..ee7d298c23 --- /dev/null +++ b/backend/.sqlx/query-f60339bdf63e72c63df45530be4e13380720237a1a310d8acf5e975189981422.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "on_behalf_of", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "f60339bdf63e72c63df45530be4e13380720237a1a310d8acf5e975189981422" +} diff --git a/backend/.sqlx/query-f720dc3321c8a351655fbb774cc954c655e4368f7e496e7d05b6beeaf091db95.json b/backend/.sqlx/query-f720dc3321c8a351655fbb774cc954c655e4368f7e496e7d05b6beeaf091db95.json new file mode 100644 index 0000000000..d5e8ddcdd5 --- /dev/null +++ b/backend/.sqlx/query-f720dc3321c8a351655fbb774cc954c655e4368f7e496e7d05b6beeaf091db95.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_node\n WHERE workspace_id = $1 AND job_id = $2 AND script_hash IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f720dc3321c8a351655fbb774cc954c655e4368f7e496e7d05b6beeaf091db95" +} diff --git a/backend/.sqlx/query-fad10d08b6a7fb4c6974a6dce79028f9bae1b9a9806c9efb2844f7e7e8196f5b.json b/backend/.sqlx/query-fad10d08b6a7fb4c6974a6dce79028f9bae1b9a9806c9efb2844f7e7e8196f5b.json new file mode 100644 index 0000000000..6d0fdd1bd8 --- /dev/null +++ b/backend/.sqlx/query-fad10d08b6a7fb4c6974a6dce79028f9bae1b9a9806c9efb2844f7e7e8196f5b.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH live AS (\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n )\n SELECT p.asset_path AS \"from_path!\", c.asset_path AS \"to_path!\"\n FROM dbt_edge e\n JOIN live l ON l.path = e.script_path\n AND (e.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND e.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = e.job_id\n -- Gated the same way as the `live` joins above, and for the same\n -- reason twice over: unpinned this folds to a plain equality the\n -- versioned key can bound, and pinned it is the only way an editor\n -- graph's NULL-to-NULL hashes meet at all.\n JOIN dbt_node p ON p.workspace_id = e.workspace_id\n AND p.script_path = e.script_path\n AND (p.script_hash = e.script_hash\n OR ($5::text IS NOT NULL AND e.script_hash IS NULL\n AND p.script_hash IS NULL))\n AND p.job_id = ch.job_id\n AND p.unique_id = e.parent_unique_id\n JOIN dbt_node c ON c.workspace_id = e.workspace_id\n AND c.script_path = e.script_path\n AND (c.script_hash = e.script_hash\n OR ($5::text IS NOT NULL AND e.script_hash IS NULL\n AND c.script_hash IS NULL))\n AND c.job_id = ch.job_id\n AND c.unique_id = e.child_unique_id\n WHERE e.workspace_id = $1\n AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL\n -- Tests attach to their model as a badge, not as a lineage edge.\n AND c.resource_type <> 'test'\n -- Scoped by the RELATIONS, like the node provenance above, not by\n -- the producing script's folder: two tables consumed in this\n -- folder but produced by a dbt project outside it would otherwise\n -- both render with their `ref()` edge missing.\n -- Same as the node scope: pinned, `asset` describes the CURRENT\n -- deploy, so gating on it drops the edges of models this version\n -- had and a later one removed. The pinned graph's own edges are\n -- the answer.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL OR EXISTS (\n SELECT 1 FROM asset a\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.path = c.asset_path\n AND ($2::text IS NULL OR a.usage_path LIKE $2)))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "from_path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "to_path!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "fad10d08b6a7fb4c6974a6dce79028f9bae1b9a9806c9efb2844f7e7e8196f5b" +} diff --git a/backend/.sqlx/query-fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179.json b/backend/.sqlx/query-fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179.json new file mode 100644 index 0000000000..d74c17a1ed --- /dev/null +++ b/backend/.sqlx/query-fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id FROM dbt_run_state\n WHERE workspace_id = $1 AND script_path = $2 AND permissioned_as = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "fb2a7f9b797a45351439746e1478545c969d995dc02aaf6673183a73cab67179" +} diff --git a/backend/.sqlx/query-fb3f9cfbe874a1f19ccf54660496532ad995ba641f0833babe5d5d597fedd55a.json b/backend/.sqlx/query-fb3f9cfbe874a1f19ccf54660496532ad995ba641f0833babe5d5d597fedd55a.json new file mode 100644 index 0000000000..d2d36be4a5 --- /dev/null +++ b/backend/.sqlx/query-fb3f9cfbe874a1f19ccf54660496532ad995ba641f0833babe5d5d597fedd55a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(\n (SELECT username FROM usr WHERE workspace_id = $1 AND email = $2),\n (SELECT COALESCE(username, email) FROM password WHERE email = $2 AND super_admin = true)\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "fb3f9cfbe874a1f19ccf54660496532ad995ba641f0833babe5d5d597fedd55a" +} diff --git a/backend/.sqlx/query-fcca85a3f80bac6c7c1e6e53bc514c050960c7f8f515c0e601ef486c68c7fcfd.json b/backend/.sqlx/query-fcca85a3f80bac6c7c1e6e53bc514c050960c7f8f515c0e601ef486c68c7fcfd.json new file mode 100644 index 0000000000..777b03cc41 --- /dev/null +++ b/backend/.sqlx/query-fcca85a3f80bac6c7c1e6e53bc514c050960c7f8f515c0e601ef486c68c7fcfd.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of)\n VALUES\n ('test-workspace', 'u/test-user/obo_member', 91001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user'),\n ('test-workspace', 'u/test-user/obo_stranger', 91002, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2'),\n ('test-workspace', 'u/test-user/obo_group', 91003, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'g/all'),\n ('test-workspace', 'u/test-user/obo_superadmin', 91004, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/ext-sa')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "fcca85a3f80bac6c7c1e6e53bc514c050960c7f8f515c0e601ef486c68c7fcfd" +} diff --git a/backend/.sqlx/query-fddab529cbb59c1dd621113a1018457546eeeaba57f5dd3abac948d60ecd952f.json b/backend/.sqlx/query-fddab529cbb59c1dd621113a1018457546eeeaba57f5dd3abac948d60ecd952f.json new file mode 100644 index 0000000000..35c401fdef --- /dev/null +++ b/backend/.sqlx/query-fddab529cbb59c1dd621113a1018457546eeeaba57f5dd3abac948d60ecd952f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest)\n VALUES ($1, $2, NULL, $3, 'd')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "fddab529cbb59c1dd621113a1018457546eeeaba57f5dd3abac948d60ecd952f" +} diff --git a/backend/.sqlx/query-ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2.json b/backend/.sqlx/query-ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2.json new file mode 100644 index 0000000000..13ed472958 --- /dev/null +++ b/backend/.sqlx/query-ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT digest FROM dbt_graph_snapshot\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3\n AND job_id = $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "digest", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ff244d58f154fc3132205fbb8374f01ceb707d9ae0b02bf382d4f5dca73829e2" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7dd3c795aa..25aefda398 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -243,11 +243,11 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "ar_archive_writer" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" dependencies = [ - "object", + "object 0.39.1", ] [[package]] @@ -803,7 +803,7 @@ checksum = "850b60ddcc664dcd848f8a2fa8436ab9336e051d6dd2b3f21f897dd8e9c24703" dependencies = [ "base64 0.22.1", "bytes", - "http 1.4.2", + "http 1.5.0", "rand 0.8.5", "reqwest 0.12.28", "serde", @@ -953,7 +953,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "hex", - "http 1.4.2", + "http 1.5.0", "ring 0.17.14", "time", "tokio", @@ -1016,7 +1016,7 @@ dependencies = [ "bytes", "bytes-utils", "fastrand 2.5.0", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "percent-encoding", "pin-project-lite", @@ -1043,7 +1043,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1118,7 +1118,7 @@ dependencies = [ "aws-types", "fastrand 2.5.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", "url", @@ -1143,7 +1143,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1189,7 +1189,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1213,7 +1213,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1257,7 +1257,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "percent-encoding", "sha2 0.10.9", "time", @@ -1300,7 +1300,7 @@ dependencies = [ "futures-core", "futures-util", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "percent-encoding", "pin-project-lite", @@ -1320,7 +1320,7 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "percent-encoding", @@ -1341,7 +1341,7 @@ dependencies = [ "h2 0.3.27", "h2 0.4.15", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", "hyper 1.11.0", @@ -1411,7 +1411,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "http-body 1.1.0", "http-body-util", @@ -1431,7 +1431,7 @@ dependencies = [ "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "pin-project-lite", "tokio", "tracing", @@ -1449,7 +1449,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "http-body 1.1.0", "http-body-util", @@ -1507,7 +1507,7 @@ dependencies = [ "axum-core 0.4.5", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "itoa", @@ -1535,7 +1535,7 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper 1.11.0", @@ -1568,7 +1568,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "mime", @@ -1587,7 +1587,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "mime", @@ -1636,7 +1636,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.37.3", "rustc-demangle", "windows-link 0.2.1", ] @@ -1907,7 +1907,7 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper 1.11.0", "hyper-named-pipe", @@ -2399,9 +2399,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.8.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" dependencies = [ "glob", "libc", @@ -2410,9 +2410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.4" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" dependencies = [ "clap_builder", "clap_derive", @@ -2420,9 +2420,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" dependencies = [ "anstream", "anstyle", @@ -3048,9 +3048,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "data-url" @@ -3889,7 +3889,7 @@ dependencies = [ "error_reporter", "h2 0.4.15", "hickory-resolver", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper 1.11.0", "hyper-rustls 0.27.9", @@ -4720,22 +4720,22 @@ dependencies = [ [[package]] name = "enum-ordinalize" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f808d588c10e464ea6f7d3eaed500049eff30aaac103460f61828c2d65b3eb" +checksum = "89dd01549b09589510cf0647475075d12071456586d70f5c75c98ae2a5537677" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" +checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -5567,7 +5567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de13e62d7e0ffc3eb40a0113ddf753cf6ec741be739164442b08893db4f9bfca" dependencies = [ "google-cloud-token", - "http 1.4.2", + "http 1.5.0", "thiserror 1.0.69", "tokio", "tokio-retry2", @@ -5688,7 +5688,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "indexmap 2.14.0", "slab", "tokio", @@ -5795,7 +5795,7 @@ dependencies = [ "base64 0.22.1", "bytes", "headers-core", - "http 1.4.2", + "http 1.5.0", "httpdate", "mime", "sha1", @@ -5807,7 +5807,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -5842,7 +5842,7 @@ checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97" dependencies = [ "dirs 6.0.0", "futures", - "http 1.4.2", + "http 1.5.0", "indicatif", "libc", "log", @@ -5976,9 +5976,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -6002,7 +6002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -6013,7 +6013,7 @@ checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "pin-project-lite", ] @@ -6039,7 +6039,7 @@ dependencies = [ "async-compression", "bstr", "futures", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper 1.11.0", "hyper-rustls 0.26.0", @@ -6100,7 +6100,7 @@ dependencies = [ "futures-channel", "futures-core", "h2 0.4.15", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "httparse", "httpdate", @@ -6120,7 +6120,7 @@ dependencies = [ "bytes", "futures-util", "headers", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "hyper-rustls 0.27.9", "hyper-util", @@ -6167,7 +6167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "hyper-util", "log", @@ -6185,7 +6185,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "hyper-util", "log", @@ -6251,7 +6251,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "hyper 1.11.0", "ipnet", @@ -6527,9 +6527,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "ipnetwork" @@ -6815,7 +6815,7 @@ dependencies = [ "either", "futures", "home", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper 1.11.0", @@ -6849,7 +6849,7 @@ dependencies = [ "chrono", "derive_more 2.1.1", "form_urlencoded", - "http 1.4.2", + "http 1.5.0", "json-patch", "k8s-openapi", "schemars 0.8.22", @@ -7066,9 +7066,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" dependencies = [ "bitflags 2.13.1", "libc", @@ -7265,19 +7265,19 @@ checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" [[package]] name = "macro_rules_attribute" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" dependencies = [ "macro_rules_attribute-proc_macro", - "paste", + "pastey", ] [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" [[package]] name = "magic-crypt" @@ -7640,7 +7640,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http 1.4.2", + "http 1.5.0", "httparse", "memchr", "mime", @@ -8147,7 +8147,7 @@ dependencies = [ "base64 0.22.1", "chrono", "getrandom 0.2.17", - "http 1.4.2", + "http 1.5.0", "rand 0.8.5", "reqwest 0.12.28", "serde", @@ -8167,6 +8167,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" version = "0.12.0" @@ -8178,7 +8187,7 @@ dependencies = [ "chrono", "form_urlencoded", "futures", - "http 1.4.2", + "http 1.5.0", "http-body-util", "httparse", "humantime", @@ -8294,7 +8303,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "hmac", - "http 1.4.2", + "http 1.5.0", "itertools 0.10.5", "log", "oauth2", @@ -8421,7 +8430,7 @@ checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" dependencies = [ "async-trait", "bytes", - "http 1.4.2", + "http 1.5.0", "opentelemetry 0.27.1", ] @@ -8433,7 +8442,7 @@ checksum = "50f6639e842a97dbea8886e3439710ae463120091e2e064518ba8e716e6ac36d" dependencies = [ "async-trait", "bytes", - "http 1.4.2", + "http 1.5.0", "opentelemetry 0.30.0", "reqwest 0.12.28", ] @@ -8446,7 +8455,7 @@ checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", - "http 1.4.2", + "http 1.5.0", "opentelemetry 0.27.1", "opentelemetry-http 0.27.0", "opentelemetry-proto 0.27.0", @@ -8465,7 +8474,7 @@ version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b" dependencies = [ - "http 1.4.2", + "http 1.5.0", "opentelemetry 0.30.0", "opentelemetry-http 0.30.0", "opentelemetry-proto 0.30.0", @@ -9453,9 +9462,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" dependencies = [ "ar_archive_writer", "cc", @@ -10041,7 +10050,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.15", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper 1.11.0", @@ -10089,7 +10098,7 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.15", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper 1.11.0", @@ -10130,7 +10139,7 @@ checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" dependencies = [ "anyhow", "async-trait", - "http 1.4.2", + "http 1.5.0", "reqwest 0.13.1", "serde", "thiserror 2.0.19", @@ -10147,7 +10156,7 @@ dependencies = [ "async-trait", "futures", "getrandom 0.2.17", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "reqwest 0.13.1", "reqwest-middleware", @@ -10253,7 +10262,7 @@ dependencies = [ "bytes", "chrono", "futures", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "oauth2", @@ -11492,9 +11501,9 @@ checksum = "1b6709c7b6754dca1311b3c73e79fcce40dd414c782c66d88e8823030093b02b" [[package]] name = "sketches-ddsketch" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" +checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56" dependencies = [ "serde", ] @@ -11899,9 +11908,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", @@ -12912,9 +12921,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -13262,7 +13271,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "httparse", "rand 0.8.5", "ring 0.17.14", @@ -13350,7 +13359,7 @@ dependencies = [ "bytes", "flate2", "h2 0.4.15", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper 1.11.0", @@ -13382,7 +13391,7 @@ dependencies = [ "base64 0.22.1", "bytes", "h2 0.4.15", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper 1.11.0", @@ -13450,7 +13459,7 @@ dependencies = [ "axum-core 0.5.6", "cookie", "futures-util", - "http 1.4.2", + "http 1.5.0", "parking_lot", "pin-project-lite", "tower-layer", @@ -13469,7 +13478,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "mime", @@ -13704,7 +13713,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.4.2", + "http 1.5.0", "httparse", "log", "native-tls", @@ -13726,7 +13735,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.4.2", + "http 1.5.0", "httparse", "log", "native-tls", @@ -14370,7 +14379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "974fa1e325e6cc5327de8887f189a441fcff4f8eedcd31ec87f0ef0cc5283fbc" dependencies = [ "bytes", - "http 1.4.2", + "http 1.5.0", "thiserror 2.0.19", "url", ] @@ -14489,7 +14498,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-nats", @@ -14574,7 +14583,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.775.2" +version = "1.777.1" dependencies = [ "async-stream", "async-trait", @@ -14587,7 +14596,7 @@ dependencies = [ "bytes", "eventsource-stream", "futures", - "http 1.4.2", + "http 1.5.0", "lazy_static", "mime_guess", "reqwest 0.13.1", @@ -14607,7 +14616,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -14620,7 +14629,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "argon2", @@ -14649,7 +14658,7 @@ dependencies = [ "git-version", "hex", "hmac", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "indexmap 2.14.0", "itertools 0.14.0", @@ -14759,11 +14768,11 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "lazy_static", "quick_cache", @@ -14782,14 +14791,16 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", "serde", "serde_json", "sqlx", + "tokio", "tracing", + "uuid", "windmill-api-auth", "windmill-common", "windmill-parser-sql-asset", @@ -14797,12 +14808,12 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "axum 0.8.9", "chrono", - "http 1.4.2", + "http 1.5.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -14823,7 +14834,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.775.2" +version = "1.777.1" dependencies = [ "reqwest 0.12.28", "serde", @@ -14833,7 +14844,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -14850,7 +14861,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14872,7 +14883,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -14895,7 +14906,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -14911,7 +14922,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -14927,12 +14938,13 @@ dependencies = [ "windmill-common", "windmill-dep-map", "windmill-git-sync", + "windmill-native-triggers", "windmill-queue", ] [[package]] name = "windmill-api-groups" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -14953,7 +14965,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -14967,7 +14979,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-nats", @@ -15002,13 +15014,13 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "axum 0.8.9", "base64 0.22.1", "chrono", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "lazy_static", "serde", @@ -15027,7 +15039,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "flate2", @@ -15045,11 +15057,11 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "axum 0.8.9", - "http 1.4.2", + "http 1.5.0", "indexmap 2.14.0", "itertools 0.14.0", "lazy_static", @@ -15067,7 +15079,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15087,12 +15099,12 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", "futures", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "itertools 0.14.0", "lazy_static", @@ -15110,6 +15122,7 @@ dependencies = [ "windmill-common", "windmill-dep-map", "windmill-git-sync", + "windmill-native-triggers", "windmill-object-store", "windmill-parser", "windmill-parser-py", @@ -15124,7 +15137,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -15152,7 +15165,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.775.2" +version = "1.777.1" dependencies = [ "lazy_static", "serde", @@ -15164,13 +15177,13 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.775.2" +version = "1.777.1" dependencies = [ "argon2", "axum 0.8.9", "chrono", "dashmap", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "lazy_static", "serde", @@ -15189,7 +15202,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", @@ -15203,13 +15216,13 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.775.2" +version = "1.777.1" dependencies = [ "axum 0.8.9", "chrono", "futures", "hex", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "lazy_static", "magic-crypt", @@ -15238,7 +15251,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.775.2" +version = "1.777.1" dependencies = [ "chrono", "lazy_static", @@ -15252,7 +15265,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "axum 0.8.9", @@ -15271,7 +15284,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.775.2" +version = "1.777.1" dependencies = [ "aes-gcm", "aho-corasick", @@ -15375,7 +15388,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.775.2" +version = "1.777.1" dependencies = [ "chrono", "itertools 0.14.0", @@ -15394,7 +15407,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.775.2" +version = "1.777.1" dependencies = [ "regex", "serde", @@ -15409,7 +15422,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15433,7 +15446,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "futures", @@ -15450,7 +15463,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.775.2" +version = "1.777.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15466,13 +15479,13 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", "chrono", "futures", - "http 1.4.2", + "http 1.5.0", "oauth2", "reqwest 0.12.28", "rmcp", @@ -15487,7 +15500,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -15496,7 +15509,7 @@ dependencies = [ "base64 0.22.1", "chrono", "hmac", - "http 1.4.2", + "http 1.5.0", "itertools 0.14.0", "lazy_static", "reqwest 0.13.1", @@ -15518,7 +15531,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "arc-swap", @@ -15543,7 +15556,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-stream", @@ -15577,7 +15590,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "futures", @@ -15595,7 +15608,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.775.2" +version = "1.777.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15604,7 +15617,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -15616,7 +15629,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde_json", @@ -15628,7 +15641,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "gosyn", @@ -15640,7 +15653,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -15652,7 +15665,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde_json", @@ -15664,7 +15677,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "nu-parser", @@ -15675,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15686,7 +15699,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15698,7 +15711,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "rustpython-ast", @@ -15709,7 +15722,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-recursion", @@ -15731,7 +15744,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde_json", @@ -15743,7 +15756,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -15757,7 +15770,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15774,7 +15787,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -15787,7 +15800,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde", @@ -15799,7 +15812,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -15817,7 +15830,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15833,7 +15846,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "rustpython-ast", @@ -15849,18 +15862,21 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", + "lazy_static", + "regex", "serde", "serde_json", + "serde_yml", "windmill-parser", "yaml-rust", ] [[package]] name = "windmill-queue" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-recursion", @@ -15899,7 +15915,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "const_format", @@ -15939,7 +15955,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.775.2" +version = "1.777.1" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15950,7 +15966,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-recursion", @@ -15959,7 +15975,7 @@ dependencies = [ "chrono", "futures", "hex", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "lazy_static", "magic-crypt", @@ -15984,7 +16000,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16008,13 +16024,13 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", "axum 0.8.9", "chrono", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "itertools 0.14.0", "lazy_static", @@ -16041,7 +16057,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16068,7 +16084,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16078,7 +16094,7 @@ dependencies = [ "chrono", "constant_time_eq 0.3.1", "hex", - "http 1.4.2", + "http 1.5.0", "itertools 0.14.0", "lazy_static", "quick_cache", @@ -16101,7 +16117,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16121,7 +16137,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16131,7 +16147,7 @@ dependencies = [ "chrono", "google-cloud-googleapis", "google-cloud-pubsub", - "http 1.4.2", + "http 1.5.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -16155,7 +16171,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16166,7 +16182,7 @@ dependencies = [ "futures", "hex", "hmac", - "http 1.4.2", + "http 1.5.0", "hyper 1.11.0", "itertools 0.14.0", "lazy_static", @@ -16191,7 +16207,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16214,7 +16230,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16238,7 +16254,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-nats", @@ -16262,7 +16278,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16297,7 +16313,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", @@ -16325,14 +16341,14 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-trait", "axum 0.8.9", "base64 0.22.1", "futures", - "http 1.4.2", + "http 1.5.0", "itertools 0.14.0", "serde", "serde_json", @@ -16350,7 +16366,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16369,7 +16385,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-once-cell", @@ -16432,6 +16448,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "serde", "serde_json", + "serde_yml", "sha2 0.10.9", "sqlx", "tar", @@ -16484,7 +16501,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.775.2" +version = "1.777.1" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a6fffe33f4..84d31776ef 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.775.2" +version = "1.777.1" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.775.2" +version = "1.777.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f008a12d4c..71ea1efdb9 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -aa05ca8e97fc8265cd724753a80db37f83243254 +59044635769f18f8ff5073236cfc7b5f41e917cc diff --git a/backend/migrations/20260725084224_add_dbt_lang.down.sql b/backend/migrations/20260725084224_add_dbt_lang.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20260725084224_add_dbt_lang.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20260725084224_add_dbt_lang.up.sql b/backend/migrations/20260725084224_add_dbt_lang.up.sql new file mode 100644 index 0000000000..ec99678460 --- /dev/null +++ b/backend/migrations/20260725084224_add_dbt_lang.up.sql @@ -0,0 +1,2 @@ +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'dbt'; +UPDATE config SET config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["dbt"]'::jsonb) WHERE name = 'worker__default' AND config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java", "duckdb", "ruby", "rlang"]}'::jsonb AND NOT config->'worker_tags' @> '"dbt"'::jsonb; diff --git a/backend/migrations/20260725084314_dbt_runtime.down.sql b/backend/migrations/20260725084314_dbt_runtime.down.sql new file mode 100644 index 0000000000..93fd7ef4b5 --- /dev/null +++ b/backend/migrations/20260725084314_dbt_runtime.down.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS dbt_run_progress; +DROP TABLE IF EXISTS dbt_run_state; +DROP TABLE IF EXISTS dbt_graph_snapshot; +DROP TABLE IF EXISTS dbt_edge; +DROP TABLE IF EXISTS dbt_node; + +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS dbt_warehouses; diff --git a/backend/migrations/20260725084314_dbt_runtime.up.sql b/backend/migrations/20260725084314_dbt_runtime.up.sql new file mode 100644 index 0000000000..c40441933d --- /dev/null +++ b/backend/migrations/20260725084314_dbt_runtime.up.sql @@ -0,0 +1,218 @@ +-- The dbt runtime's tables: the parsed manifest that becomes the asset graph, +-- the retry state a `dbt retry` resumes from, and a run's live per-model +-- progress. + +-- Physical-relation asset kind. Identity is the relation itself +-- (`//`, the warehouse named as the workspace +-- configures it), never the producing tool, so a dbt mart and a native script +-- reading the same table resolve to one node and the lineage is one graph +-- across the boundary (docs/dbt-runtime.md, decision 11). +ALTER TYPE ASSET_KIND ADD VALUE IF NOT EXISTS 'dbt'; + +-- Warehouses configured once for the workspace, so a dbt project needs no +-- connection knowledge to run: the descriptor names one by NAME (or nothing, for +-- `main`) and the value here points at the RESOURCE that holds the credentials, +-- exactly as `large_file_storage` does for buckets. A flat map keyed by name, +-- which is also what asset identity keys on. +-- +-- {"main": {"resource_path": "$res:u/admin/wh", "target": "prod"}, +-- "eu": {"resource_path": "$res:u/admin/wh_eu"}} +ALTER TABLE workspace_settings ADD COLUMN IF NOT EXISTS dbt_warehouses JSONB; + +-- Parsed dbt manifest, one row per dbt node. The full manifest.json is not +-- stored -- only the fields the asset graph renders. +-- +-- Keyed by (path, version, job). The VERSION because each deployed version of a +-- script keeps its own graph, so a finished run can be shown the project as it +-- was rather than whatever is deployed today. The JOB because a dynamic +-- descriptor -- a `{{ }}` placeholder in `vars` -- can resolve to a different +-- set of models per run, and those runs re-ingest; the zero UUID means "the +-- version's own graph, as deployed", which is what a static descriptor writes +-- once and every one of its runs reads. A sentinel rather than NULL because +-- `job_id` is part of the key and Postgres does not treat two NULLs as the same +-- key, so each re-ingest would add a row set instead of replacing one. +CREATE TABLE IF NOT EXISTS dbt_node ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT NOT NULL, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + -- dbt's own node id, e.g. `model.jaffle_shop.customers`. Stable across runs + -- and the join key for dbt_edge and for run_results ingestion. + unique_id TEXT NOT NULL, + resource_type TEXT NOT NULL, + name TEXT NOT NULL, + -- The physical relation this node produces, spelled as the matching `asset` + -- row's path (`//`). NULL for nodes with no + -- relation: ephemeral models, tests. + asset_path TEXT, + materialized TEXT, + -- Windmill's equivalent write strategy (replace | append | merge | scd2), + -- NULL when dbt's materialization has no analogue (view, ephemeral). + materialize_strategy TEXT, + unique_key TEXT, + tags TEXT[] NOT NULL DEFAULT '{}', + description TEXT, + -- Test nodes only: the generic test name (`unique`, `not_null`, + -- `accepted_values`, `relationships`, or a package/custom test's own name), + -- the column it is attached to, its rendered kwargs and dbt's severity. + -- Severity is stored as dbt spells it; readers compare case-insensitively + -- because dbt-core 1.x echoes the author's casing while 2.x uppercases it. + test_kind TEXT, + test_column TEXT, + test_args JSONB, + severity TEXT, + attached_node TEXT, + -- Declared per-column metadata (name -> description), the column sets the + -- asset graph shows. dbt's manifest carries no column-to-column lineage, so + -- this is not a column lineage graph. + columns JSONB, + freshness JSONB, + -- The model's SQL as written, for the graph to render. `dbt parse` fills it; + -- `compiled_code` would need a `dbt compile`, which no phase runs. + raw_code TEXT, + -- Its path inside the dbt project, e.g. `models/staging/stg_orders.sql`. + original_file_path TEXT, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, script_hash, job_id, unique_id), + -- Composite because `script`'s key is (workspace_id, hash). A version's graph + -- dies with the version and nothing has to sweep it. + CONSTRAINT dbt_node_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- `ref()` / `source()` lineage, straight from the manifest's parent_map. +CREATE TABLE IF NOT EXISTS dbt_edge ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT NOT NULL, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + parent_unique_id TEXT NOT NULL, + child_unique_id TEXT NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, script_hash, job_id, parent_unique_id, child_unique_id), + CONSTRAINT dbt_edge_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- One row per stored graph, so a snapshot's EXISTENCE does not depend on it +-- having any nodes: a dynamic run can disable every model, and that empty graph +-- has to be distinguishable from a run that stored nothing -- otherwise the run +-- page falls back to the deployed models and shows what the run never built. +-- +-- It is also where the content digest lives, once. A run whose digest matches +-- the version's writes no snapshot at all: marking a descriptor dynamic is +-- conservative (a `{{ }}` in `vars` says the arguments reach dbt, not that they +-- change which models exist), so the usual dynamic run resolves to exactly the +-- graph the deploy stored. +CREATE TABLE IF NOT EXISTS dbt_graph_snapshot ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT NOT NULL, + job_id UUID NOT NULL, + digest TEXT NOT NULL, + -- On the DEPLOYED row only: the relation root the last ingest to write this + -- row resolved. Written by every ingest that is not a run's own snapshot, + -- including one that publishes no ownership — a version that cannot claim the + -- path would otherwise record nothing and compare against a stale root. + -- + -- The drift check needs "where do the current usages point", and no other + -- row answers it: the deploy's own root goes stale the moment a run at a + -- moved profile republishes, and "the newest ingest" is wrong because a run + -- whose graph matches the deploy's stores nothing at all. + relation_root_at_last_ingest TEXT, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, script_hash, job_id), + -- Same cascade as the rows it stands for. A marker outliving its nodes is + -- read as a snapshot that has none, and its digest still answers the + -- suppression check. + CONSTRAINT dbt_graph_snapshot_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- What a `dbt retry` resumes from. Keyed by path, not by version: there is one +-- saved run per script, and `identity` -- the project digest, warehouse and +-- engine -- is what refuses a resume that no longer describes the same run. +CREATE TABLE IF NOT EXISTS dbt_run_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + -- Part of the key: a retry replaces the caller's arguments with the saved + -- ones, so state written by one principal must not be restorable by another + -- — that would hand them the literal `select` and `vars` of a run they were + -- never entitled to see. + permissioned_as VARCHAR(255) NOT NULL, + identity TEXT NOT NULL, + -- The invocation's job arguments. `dbt retry` reuses the original selection + -- and vars, so the graph refresh and the build must agree with them rather + -- than with the retry request's. + args JSONB NOT NULL DEFAULT '{}'::jsonb, + -- Text rather than jsonb: nothing queries inside it, and this is highly + -- repetitive JSON that TOAST's compression handles well (about nine to one). + run_results TEXT NOT NULL, + job_id UUID, + -- Whether those results hold a node `dbt retry` would rebuild. A run that + -- failed before building anything, or succeeded outright, is still saved — + -- restoring it is how a retry can say the run succeeded rather than that no + -- state exists — but nothing may offer it as resumable. + retryable BOOLEAN NOT NULL DEFAULT false, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, permissioned_as) +); + +-- Live per-model progress for one RUN. +-- +-- `materialized_partition` cannot answer this: its key is the relation, one row +-- per table, and its `job_id` names only the last writer. Two runs of one +-- project building the same models take that row from each other, so a progress +-- read filtered by job loses nodes and flickers between states. That table is +-- left as it is -- the current state of a relation is what the pipeline canvas +-- and fork defer read, and one row per relation is right for them. +CREATE TABLE IF NOT EXISTS dbt_run_progress ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + job_id UUID NOT NULL, + asset_kind ASSET_KIND NOT NULL, + asset_path VARCHAR(255) NOT NULL, + status MATERIALIZATION_STATUS NOT NULL, + row_count BIGINT, + error TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, job_id, asset_kind, asset_path) +); + +-- Resolve a physical relation back to the dbt node that produces it: the asset +-- graph renders from `asset` rows and needs the dbt provenance per node. +CREATE INDEX IF NOT EXISTS idx_dbt_node_asset_path + ON dbt_node (workspace_id, asset_path) WHERE asset_path IS NOT NULL; + +-- No foreign key from the per-run rows to `v2_job`: three were deliberately +-- dropped from it for the write amplification they cost on the hottest table in +-- the system. Growth is bounded by age instead and pruned by the dbt runs +-- themselves, so no background sweep has to learn about these tables -- hence +-- an index per age sweep, and one per "does this job have a snapshot" lookup. +CREATE INDEX IF NOT EXISTS idx_dbt_node_job ON dbt_node (job_id) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_node_run_age ON dbt_node (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_edge_run_age ON dbt_edge (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_graph_snapshot_job + ON dbt_graph_snapshot (workspace_id, job_id); +CREATE INDEX IF NOT EXISTS idx_dbt_graph_snapshot_age + ON dbt_graph_snapshot (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; +CREATE INDEX IF NOT EXISTS idx_dbt_run_progress_updated_at + ON dbt_run_progress (updated_at); + +-- All of these are written on a user_db transaction (SET LOCAL ROLE +-- windmill_user/windmill_admin), so they must be granted explicitly rather than +-- relying on ALTER DEFAULT PRIVILEGES, which only covers objects created by the +-- role that set them (see 20260720131744). +GRANT ALL ON dbt_node TO windmill_user; +GRANT ALL ON dbt_node TO windmill_admin; +GRANT ALL ON dbt_edge TO windmill_user; +GRANT ALL ON dbt_edge TO windmill_admin; +GRANT ALL ON dbt_graph_snapshot TO windmill_user; +GRANT ALL ON dbt_graph_snapshot TO windmill_admin; +GRANT ALL ON dbt_run_state TO windmill_user; +GRANT ALL ON dbt_run_state TO windmill_admin; +GRANT ALL ON dbt_run_progress TO windmill_user; +GRANT ALL ON dbt_run_progress TO windmill_admin; diff --git a/backend/migrations/20260727145314_materialized_partition_job_id_index.down.sql b/backend/migrations/20260727145314_materialized_partition_job_id_index.down.sql new file mode 100644 index 0000000000..9b33bff978 --- /dev/null +++ b/backend/migrations/20260727145314_materialized_partition_job_id_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_materialized_partition_job; diff --git a/backend/migrations/20260727145314_materialized_partition_job_id_index.up.sql b/backend/migrations/20260727145314_materialized_partition_job_id_index.up.sql new file mode 100644 index 0000000000..6bcdc246ae --- /dev/null +++ b/backend/migrations/20260727145314_materialized_partition_job_id_index.up.sql @@ -0,0 +1,9 @@ +-- `job_id` leads neither the primary key nor the asset/status index, both shaped +-- for the per-relation lookups that came first — but the closing sweep a dbt run +-- makes (`terminalize_running_relations`) reads and writes by it, settling the +-- models a cancelled or timed-out run left `running`. +-- Small table until a partitioned asset is backfilled, then one row per slice +-- per asset. +CREATE INDEX IF NOT EXISTS idx_materialized_partition_job + ON materialized_partition (workspace_id, job_id) + WHERE job_id IS NOT NULL; diff --git a/backend/migrations/20260731161812_backfill_datatablemigration_git_sync_include_type.down.sql b/backend/migrations/20260731161812_backfill_datatablemigration_git_sync_include_type.down.sql new file mode 100644 index 0000000000..e1b90875dc --- /dev/null +++ b/backend/migrations/20260731161812_backfill_datatablemigration_git_sync_include_type.down.sql @@ -0,0 +1,37 @@ +-- Strip `datatablemigration` back out of every include_type, at both levels. +-- Lossy in the same way the up is: a config that had opted in deliberately is +-- indistinguishable from one the backfill touched. + +UPDATE workspace_settings +SET git_sync = jsonb_set( + git_sync, + '{include_type}', + (git_sync->'include_type') - 'datatablemigration' + ) +WHERE jsonb_typeof(git_sync->'include_type') = 'array' + AND git_sync->'include_type' @> '"datatablemigration"'::jsonb; + +UPDATE workspace_settings ws +SET git_sync = jsonb_set(ws.git_sync, '{repositories}', updated.repositories) +FROM ( + SELECT + s.workspace_id, + jsonb_agg( + CASE + WHEN repo->'settings'->'include_type' @> '"datatablemigration"'::jsonb + THEN jsonb_set( + repo, + '{settings,include_type}', + (repo->'settings'->'include_type') - 'datatablemigration' + ) + ELSE repo + END + ORDER BY idx + ) AS repositories + FROM workspace_settings s, + LATERAL jsonb_array_elements(s.git_sync->'repositories') WITH ORDINALITY AS t(repo, idx) + WHERE jsonb_typeof(s.git_sync->'repositories') = 'array' + GROUP BY s.workspace_id +) AS updated +WHERE ws.workspace_id = updated.workspace_id + AND ws.git_sync->'repositories' IS DISTINCT FROM updated.repositories; diff --git a/backend/migrations/20260731161812_backfill_datatablemigration_git_sync_include_type.up.sql b/backend/migrations/20260731161812_backfill_datatablemigration_git_sync_include_type.up.sql new file mode 100644 index 0000000000..3ec18bf607 --- /dev/null +++ b/backend/migrations/20260731161812_backfill_datatablemigration_git_sync_include_type.up.sql @@ -0,0 +1,45 @@ +-- Data table SQL migrations became a git-sync object type (`datatablemigration`). +-- Existing settings were written before it existed, so every workspace would read +-- as "migrations opted out" until someone re-saved the form. Opt in the configs +-- that already sync something, at both levels an include_type can live: the +-- workspace-level default and each repository's own settings. +-- +-- An absent or empty include_type means "inherit / not configured yet" and is left +-- alone; the defaults the UI and CLI now write already carry the new type. + +UPDATE workspace_settings +SET git_sync = jsonb_set( + git_sync, + '{include_type}', + (git_sync->'include_type') || '"datatablemigration"'::jsonb + ) +WHERE jsonb_typeof(git_sync->'include_type') = 'array' + AND jsonb_array_length(git_sync->'include_type') > 0 + AND NOT (git_sync->'include_type' @> '"datatablemigration"'::jsonb); + +UPDATE workspace_settings ws +SET git_sync = jsonb_set(ws.git_sync, '{repositories}', updated.repositories) +FROM ( + SELECT + s.workspace_id, + jsonb_agg( + CASE + WHEN jsonb_typeof(repo->'settings'->'include_type') = 'array' + AND jsonb_array_length(repo->'settings'->'include_type') > 0 + AND NOT (repo->'settings'->'include_type' @> '"datatablemigration"'::jsonb) + THEN jsonb_set( + repo, + '{settings,include_type}', + (repo->'settings'->'include_type') || '"datatablemigration"'::jsonb + ) + ELSE repo + END + ORDER BY idx + ) AS repositories + FROM workspace_settings s, + LATERAL jsonb_array_elements(s.git_sync->'repositories') WITH ORDINALITY AS t(repo, idx) + WHERE jsonb_typeof(s.git_sync->'repositories') = 'array' + GROUP BY s.workspace_id +) AS updated +WHERE ws.workspace_id = updated.workspace_id + AND ws.git_sync->'repositories' IS DISTINCT FROM updated.repositories; diff --git a/backend/migrations/20260731220637_on_behalf_of.down.sql b/backend/migrations/20260731220637_on_behalf_of.down.sql new file mode 100644 index 0000000000..f4947b27bf --- /dev/null +++ b/backend/migrations/20260731220637_on_behalf_of.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +ALTER TABLE script DROP COLUMN IF EXISTS on_behalf_of; +ALTER TABLE flow DROP COLUMN IF EXISTS on_behalf_of; diff --git a/backend/migrations/20260731220637_on_behalf_of.up.sql b/backend/migrations/20260731220637_on_behalf_of.up.sql new file mode 100644 index 0000000000..ca2c6fd884 --- /dev/null +++ b/backend/migrations/20260731220637_on_behalf_of.up.sql @@ -0,0 +1,6 @@ +-- Add up migration script here +-- Authorization identity a script/flow runs as. NULL means "not recorded"; the migration that +-- follows this one backfills every row that had an on_behalf_of_email, so by the end of this +-- release a NULL principal means the runnable runs as its caller. +ALTER TABLE script ADD COLUMN IF NOT EXISTS on_behalf_of VARCHAR(255); +ALTER TABLE flow ADD COLUMN IF NOT EXISTS on_behalf_of VARCHAR(255); diff --git a/backend/migrations/20260801043001_backfill_on_behalf_of.down.sql b/backend/migrations/20260801043001_backfill_on_behalf_of.down.sql new file mode 100644 index 0000000000..bcd8d2cbd5 --- /dev/null +++ b/backend/migrations/20260801043001_backfill_on_behalf_of.down.sql @@ -0,0 +1,33 @@ +-- Add down migration script here +-- The column was never dropped, so this only re-derives it from the principal for the rows the +-- up migration backfilled. One naming a group gets that group's synthetic address, which is what +-- the column held before. + +CREATE FUNCTION pg_temp.email_from_permissioned_as(w_id VARCHAR, permissioned_as VARCHAR) +RETURNS VARCHAR AS $$ + -- Prefix first, as every reader decides: a group name or an email-shaped username may + -- itself contain '@', so only an unprefixed value is an address. + SELECT CASE + WHEN $2 LIKE 'g/%' + THEN 'group-' || substring($2 from 3) || '@windmill.dev' + -- Mirrors the up migration's `u/` arm, superadmin fallback included: one acting + -- outside their workspaces has no usr row, and losing their address here would + -- leave the previous runtime unable to authenticate the runnable. + WHEN $2 LIKE 'u/%' + THEN COALESCE( + (SELECT u.email FROM usr u WHERE u.workspace_id = $1 AND u.username = substring($2 from 3)), + (SELECT p.email FROM password p WHERE p.super_admin + AND (p.username = substring($2 from 3) OR p.email = substring($2 from 3)) + ORDER BY p.email LIMIT 1), + substring($2 from 3) || '@unknown.windmill.dev') + ELSE $2 + END; +$$ LANGUAGE SQL STABLE; + +UPDATE script SET on_behalf_of_email = + pg_temp.email_from_permissioned_as(workspace_id, on_behalf_of) + WHERE on_behalf_of IS NOT NULL; + +UPDATE flow SET on_behalf_of_email = + pg_temp.email_from_permissioned_as(workspace_id, on_behalf_of) + WHERE on_behalf_of IS NOT NULL; diff --git a/backend/migrations/20260801043001_backfill_on_behalf_of.up.sql b/backend/migrations/20260801043001_backfill_on_behalf_of.up.sql new file mode 100644 index 0000000000..fc27c23999 --- /dev/null +++ b/backend/migrations/20260801043001_backfill_on_behalf_of.up.sql @@ -0,0 +1,63 @@ +-- Add up migration script here +-- The permissioned_as becomes the identity this release reads; the email is derived from it at +-- read time, as triggers have always done. Backfilling it also retires the fallback to +-- created_by / edited_by, so a runnable deployed before the column existed starts running +-- as the user its on_behalf_of_email always named rather than as whoever last deployed it. +-- +-- `on_behalf_of_email` stays for now: a worker predating this release resolves a script or flow +-- through `get_script_info_for_hash` / `get_latest_hash_for_path`, which select that column, and +-- workers are expected to lag the server. Deploys keep writing it until every live worker is new +-- (MIN_VERSION_SUPPORTS_ON_BEHALF_OF_PRINCIPAL); a later release stops writing it and drops it. +-- +-- That later migration MUST re-run this backfill before dropping: server pods are mixed for the +-- minute or two a rollout takes, and one still on the previous release writes only the address — +-- leaving a runnable deployed in that window with no principal, which reads as no identity at +-- all and runs it as its caller. Re-deriving picks those up; dropping without it makes them +-- permanent. + +-- Mirrors `users::permissioned_as_from_email`: a real account wins over the synthetic group +-- namespace, which is not reserved and may be a user's own address. +CREATE FUNCTION pg_temp.permissioned_as_from_email(w_id VARCHAR, email VARCHAR) +RETURNS VARCHAR AS $$ + SELECT COALESCE( + -- `username_to_permissioned_as`: an email-shaped username is its own principal unless + -- it contains a slash, which a reader would split on. + (SELECT CASE WHEN u.username LIKE '%@%' AND u.username NOT LIKE '%/%' THEN u.username + ELSE 'u/' || u.username END + FROM usr u WHERE u.workspace_id = $1 AND u.email = $2), + -- A superadmin acting outside their workspaces has no usr row. + (SELECT CASE WHEN COALESCE(p.username, p.email) LIKE '%@%' + AND COALESCE(p.username, p.email) NOT LIKE '%/%' + THEN COALESCE(p.username, p.email) + ELSE 'u/' || COALESCE(p.username, p.email) END + FROM password p WHERE p.email = $2 AND p.super_admin), + (SELECT 'g/' || g.name FROM group_ g + WHERE g.workspace_id = $1 + AND $2 = 'group-' || g.name || '@windmill.dev') + ); +$$ LANGUAGE SQL STABLE; + +-- A principal wider than `v2_job.permissioned_as` could not be enqueued, so it is not recorded +-- at all — the runnable falls back to running as its caller until someone picks an identity the +-- deploy path accepts. Only an address-derived principal can reach that width; a username is +-- capped at 50. +UPDATE script SET on_behalf_of = + pg_temp.permissioned_as_from_email(workspace_id, on_behalf_of_email) + WHERE on_behalf_of_email IS NOT NULL AND on_behalf_of IS NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, on_behalf_of_email)) <= 55; + +UPDATE flow SET on_behalf_of = + pg_temp.permissioned_as_from_email(workspace_id, on_behalf_of_email) + WHERE on_behalf_of_email IS NOT NULL AND on_behalf_of IS NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, on_behalf_of_email)) <= 55; + +-- Drafts carry the same pair in their value. +UPDATE draft SET value = to_json(jsonb_set( + to_jsonb(value), + ARRAY['on_behalf_of'], + to_jsonb(pg_temp.permissioned_as_from_email(workspace_id, value->>'on_behalf_of_email')))) + WHERE typ IN ('script', 'flow') + AND value->>'on_behalf_of_email' IS NOT NULL + AND value->>'on_behalf_of' IS NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, value->>'on_behalf_of_email')) <= 55; + diff --git a/backend/migrations/20260801060114_drop_azure_trigger_email.down.sql b/backend/migrations/20260801060114_drop_azure_trigger_email.down.sql new file mode 100644 index 0000000000..cf51c37487 --- /dev/null +++ b/backend/migrations/20260801060114_drop_azure_trigger_email.down.sql @@ -0,0 +1,23 @@ +-- Rebuild email from permissioned_as, mirroring +-- windmill_common::users::get_email_from_permissioned_as. + +ALTER TABLE azure_trigger ADD COLUMN email VARCHAR(255) NOT NULL DEFAULT ''; + +UPDATE azure_trigger t SET email = CASE + WHEN t.permissioned_as LIKE 'u/%' THEN COALESCE( + (SELECT u.email FROM usr u + WHERE u.workspace_id = t.workspace_id + AND u.username = SUBSTRING(t.permissioned_as FROM 3)), + (SELECT p.email FROM password p + WHERE p.super_admin + AND (p.username = SUBSTRING(t.permissioned_as FROM 3) + OR p.email = SUBSTRING(t.permissioned_as FROM 3)) + LIMIT 1), + SUBSTRING(t.permissioned_as FROM 3) || '@unknown.windmill.dev' + ) + WHEN t.permissioned_as LIKE 'g/%' + THEN 'group-' || SUBSTRING(t.permissioned_as FROM 3) || '@windmill.dev' + ELSE t.permissioned_as +END; + +ALTER TABLE azure_trigger ALTER COLUMN email DROP DEFAULT; diff --git a/backend/migrations/20260801060114_drop_azure_trigger_email.up.sql b/backend/migrations/20260801060114_drop_azure_trigger_email.up.sql new file mode 100644 index 0000000000..293822e25c --- /dev/null +++ b/backend/migrations/20260801060114_drop_azure_trigger_email.up.sql @@ -0,0 +1,4 @@ +-- azure_trigger.email duplicated an address that permissioned_as already +-- determines, and which every other trigger table derives at fire time. + +ALTER TABLE azure_trigger DROP COLUMN email; diff --git a/backend/migrations/20260801121717_dbt_editor_preview_graph.down.sql b/backend/migrations/20260801121717_dbt_editor_preview_graph.down.sql new file mode 100644 index 0000000000..ada8fa319c --- /dev/null +++ b/backend/migrations/20260801121717_dbt_editor_preview_graph.down.sql @@ -0,0 +1,26 @@ +-- The editor graphs go first: they are exactly the rows the restored NOT NULL +-- would reject, and they are throwaway by construction. +DELETE FROM dbt_node WHERE script_hash IS NULL; +DELETE FROM dbt_edge WHERE script_hash IS NULL; +DELETE FROM dbt_graph_snapshot WHERE script_hash IS NULL; + +DROP INDEX IF EXISTS idx_dbt_graph_snapshot_editor_path; +ALTER TABLE dbt_graph_snapshot DROP COLUMN IF EXISTS permissioned_as; + +DROP INDEX IF EXISTS dbt_node_editor_key; +DROP INDEX IF EXISTS dbt_node_versioned_key; +ALTER TABLE dbt_node ALTER COLUMN script_hash SET NOT NULL; +ALTER TABLE dbt_node ADD CONSTRAINT dbt_node_pkey + PRIMARY KEY (workspace_id, script_path, script_hash, job_id, unique_id); + +DROP INDEX IF EXISTS dbt_edge_editor_key; +DROP INDEX IF EXISTS dbt_edge_versioned_key; +ALTER TABLE dbt_edge ALTER COLUMN script_hash SET NOT NULL; +ALTER TABLE dbt_edge ADD CONSTRAINT dbt_edge_pkey + PRIMARY KEY (workspace_id, script_path, script_hash, job_id, parent_unique_id, child_unique_id); + +DROP INDEX IF EXISTS dbt_graph_snapshot_editor_key; +DROP INDEX IF EXISTS dbt_graph_snapshot_versioned_key; +ALTER TABLE dbt_graph_snapshot ALTER COLUMN script_hash SET NOT NULL; +ALTER TABLE dbt_graph_snapshot ADD CONSTRAINT dbt_graph_snapshot_pkey + PRIMARY KEY (workspace_id, script_path, script_hash, job_id); diff --git a/backend/migrations/20260801121717_dbt_editor_preview_graph.up.sql b/backend/migrations/20260801121717_dbt_editor_preview_graph.up.sql new file mode 100644 index 0000000000..b43950517b --- /dev/null +++ b/backend/migrations/20260801121717_dbt_editor_preview_graph.up.sql @@ -0,0 +1,62 @@ +-- A graph parsed from the EDITOR's buffer, which names no deployed version. +-- +-- The two provenances that existed keyed on one: a version's own graph (the +-- zero-UUID `job_id`) and a run's snapshot of a version. A buffer refresh is +-- neither. It cannot borrow the deployed version's hash — the buffer differs +-- from it, which is the point, and a project being written has no deployed +-- version at all — so `script_hash` becomes NULL and the graph is keyed to the +-- preview job that parsed it, readable only back through that job and never +-- through the path. +-- +-- The foreign key to `script` is left exactly as written: it is MATCH SIMPLE, +-- so a NULL in either column satisfies it. A versioned row still dies with its +-- version and a version-less one is simply outside its reach. +-- +-- Two partial unique indexes rather than one with NULLS NOT DISTINCT, which +-- needs Postgres 15: a versioned graph is keyed by its version, a buffer parse +-- by its job alone (a preview job id is unique on its own). +ALTER TABLE dbt_node DROP CONSTRAINT dbt_node_pkey; +ALTER TABLE dbt_node ALTER COLUMN script_hash DROP NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_node_versioned_key + ON dbt_node (workspace_id, script_path, script_hash, job_id, unique_id) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_node_editor_key + ON dbt_node (workspace_id, job_id, unique_id) + WHERE script_hash IS NULL; + +ALTER TABLE dbt_edge DROP CONSTRAINT dbt_edge_pkey; +ALTER TABLE dbt_edge ALTER COLUMN script_hash DROP NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_edge_versioned_key + ON dbt_edge (workspace_id, script_path, script_hash, job_id, parent_unique_id, child_unique_id) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_edge_editor_key + ON dbt_edge (workspace_id, job_id, parent_unique_id, child_unique_id) + WHERE script_hash IS NULL; + +ALTER TABLE dbt_graph_snapshot DROP CONSTRAINT dbt_graph_snapshot_pkey; +ALTER TABLE dbt_graph_snapshot ALTER COLUMN script_hash DROP NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_graph_snapshot_versioned_key + ON dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_graph_snapshot_editor_key + ON dbt_graph_snapshot (workspace_id, job_id) + WHERE script_hash IS NULL; + +-- On an EDITOR graph only: the principal whose parse wrote it. +-- +-- Retention for these is a count per script rather than a clock, and the scope +-- has to be something the caller cannot choose. A preview's path IS the +-- caller's to choose and needs only `jobs:run`, so bounding by path alone would +-- let one caller's parses evict the graphs of whoever is actually editing that +-- script. `permissioned_as` is the execution principal the queue derived, which +-- is the same reason `dbt_run_state` keys on it. +ALTER TABLE dbt_graph_snapshot ADD COLUMN IF NOT EXISTS permissioned_as VARCHAR(255); + +-- An editor graph is bounded per (path, principal) rather than by age: the +-- newest few are kept and the rest dropped as each refresh lands, so this is the +-- index that ordering reads. The instance-wide age sweep every dbt run already +-- performs (`job_id <> DEPLOYED_GRAPH`) still catches one refreshed once and +-- left. +CREATE INDEX IF NOT EXISTS idx_dbt_graph_snapshot_editor_path + ON dbt_graph_snapshot (workspace_id, script_path, permissioned_as, ingested_at) + WHERE script_hash IS NULL; diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index b8a12fb476..9c70b93d64 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -273,7 +273,7 @@ impl AssetsFinder { Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { let path = parse_asset_syntax(&value, false) .map(|(_, p)| p) - .unwrap_or(&value); + .unwrap_or(std::borrow::Cow::Borrowed(value)); self.assets.push(ParseAssetsResult { kind, path: path.to_string(), diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 0e4e12eb89..6c87c3705f 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.775.2" +version = "1.777.1" dependencies = [ "aho-corasick", "anyhow", @@ -6227,6 +6227,7 @@ dependencies = [ "jsonwebtoken", "lazy_static", "magic-crypt", + "memchr", "native-tls", "once_cell", "pep440_rs", @@ -6266,13 +6267,14 @@ dependencies = [ "windmill-macros", "windmill-parser", "windmill-parser-sql", + "windmill-parser-sql-asset", "windmill-parser-ts", "windmill-types", ] [[package]] name = "windmill-macros" -version = "1.775.2" +version = "1.777.1" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.775.2" +version = "1.777.1" dependencies = [ "convert_case", "serde", @@ -6293,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde", @@ -6488,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,18 +6572,21 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", + "lazy_static", + "regex", "serde", "serde_json", + "serde_yml", "windmill-parser", "yaml-rust", ] [[package]] name = "windmill-types" -version = "1.775.2" +version = "1.777.1" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1cb5004022..a1c4725652 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.775.2" +version = "1.777.1" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index c843652e26..59e963d52c 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -164,6 +164,14 @@ pub fn parse_ansible(code: &str) -> String { wrap_sig(windmill_parser_yaml::parse_ansible_sig(code)) } +/// Shares the ansible parser's feature: both live in windmill-parser-yaml, and +/// a dbt descriptor is YAML too. +#[cfg(feature = "ansible-parser")] +#[wasm_bindgen] +pub fn parse_dbt(code: &str) -> String { + wrap_sig(windmill_parser_yaml::parse_dbt_sig(code)) +} + #[cfg(feature = "ansible-parser")] #[wasm_bindgen] pub fn parse_ansible_delegate(code: &str) -> String { diff --git a/backend/parsers/windmill-parser-yaml/Cargo.toml b/backend/parsers/windmill-parser-yaml/Cargo.toml index eb01ba1936..eb2f16fccc 100644 --- a/backend/parsers/windmill-parser-yaml/Cargo.toml +++ b/backend/parsers/windmill-parser-yaml/Cargo.toml @@ -14,3 +14,6 @@ windmill-parser.workspace = true anyhow.workspace = true serde_json.workspace = true serde.workspace = true +serde_yml.workspace = true +regex.workspace = true +lazy_static.workspace = true diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs new file mode 100644 index 0000000000..24f1e2c39d --- /dev/null +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -0,0 +1,781 @@ +//! The `ScriptLang::Dbt` script artifact: a YAML descriptor holding a dbt +//! project's run configuration. The project itself is the script's module +//! bundle, so the descriptor names no source for it. +//! +//! Field names track dbt's and astronomer-cosmos's vocabulary so the mental +//! model ports without translation (docs/dbt-runtime.md, decision 22). +//! `select` / `exclude` / `selector` are passed to dbt **verbatim**: the +//! selector grammar is dbt's, and reimplementing it is a standing source of +//! divergence. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use windmill_parser::{Arg, MainArgSignature, Typ}; + +/// Which dbt to run. The shipped default is `dbt-core-1x` because it runs +/// today's projects untouched; `fusion` is never bundled and is fetched from +/// dbt Labs at runtime (docs/dbt-runtime.md, decision 1). +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "kebab-case")] +pub enum DbtEngine { + #[default] + #[serde(rename = "dbt-core-1x")] + DbtCore1x, + #[serde(rename = "dbt-core-2x")] + DbtCore2x, + Fusion, +} + +impl DbtEngine { + pub fn as_str(&self) -> &'static str { + match self { + DbtEngine::DbtCore1x => "dbt-core-1x", + DbtEngine::DbtCore2x => "dbt-core-2x", + DbtEngine::Fusion => "fusion", + } + } + + /// The level the machine-readable file log is written at. Separate from the + /// console log, which always stays human-readable at the default level and + /// is what reaches the job log. + pub fn progress_log_level(&self) -> &'static str { + match self { + DbtEngine::DbtCore1x => "info", + DbtEngine::DbtCore2x | DbtEngine::Fusion => "debug", + } + } + + /// Whether the engine writes per-node events to its JSON **file** log — the + /// only source of *live* per-model status. + /// + /// Not a claim that the others produce none: dbt-core 2.x and Fusion emit + /// the same events, on the console, and ignore `--log-format-file json` + /// though both accept it. Reading them would mean taking over the console + /// and re-rendering the job log. Flip this the moment either honours the + /// flag — nothing else has to change (docs/dbt-runtime.md, "Live per-model + /// progress"). + pub fn emits_node_events(&self) -> bool { + matches!(self, DbtEngine::DbtCore1x) + } +} + +/// How the warehouse connection is supplied. Both paths are supported +/// (decision 8): render `profiles.yml` from a Windmill resource, or keep the +/// project's own file and inject Windmill secrets as env vars for +/// `{{ env_var() }}`. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(deny_unknown_fields)] +pub struct DbtProfile { + /// A warehouse configured on the workspace, by name; omitted takes the + /// default one. This is the ONLY way a project names a warehouse — there is + /// no per-descriptor resource — so a dbt project carries no connection at + /// all, the same bargain `s3://` and `ducklake://` make with workspace + /// storage, and asset identity has exactly one spelling to key on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub warehouse: Option, + + /// dbt target name. It selects which output of the profile runs; the + /// `` component of asset identity comes from `warehouse` above. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target: Option, + /// Path (relative to `project`) of the project's own `profiles.yml`, used + /// instead of rendering one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profiles_yml: Option, + /// Target schema (BigQuery calls it the dataset). Required for adapters + /// whose Windmill resource carries none — a BigQuery resource is a raw + /// service-account JSON, which has no dataset in it. Overrides the + /// resource's own value where there is one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// dbt adapter (`postgres` | `snowflake` | `bigquery` | `databricks`), + /// spelled as dbt's own `type:`. Optional: the worker infers it from the + /// resource's shape, and this pins it when the inference is wrong or the + /// resource is a custom type. + #[serde(default, rename = "type", skip_serializing_if = "Option::is_none")] + pub adapter: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(rename_all = "snake_case")] +pub enum DbtTestBehavior { + /// `dbt build` — models and their tests interleaved, a model's tests gating + /// its children. dbt's own default and the only behavior that stops bad + /// data propagating mid-run. + #[default] + Build, + /// `dbt run` then `dbt test`. + AfterAll, + /// `dbt run` only. + None, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(deny_unknown_fields)] +pub struct DbtDescriptor { + #[serde(default)] + pub engine: Option, + #[serde(default)] + pub profile: DbtProfile, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub select: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, + /// A named selector from the project's `selectors.yml`, passed verbatim. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + #[serde(default)] + pub test_behavior: DbtTestBehavior, + /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are + /// all normal — so values keep their YAML type; only string leaves carry + /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing + /// everything to a string would make a `false` var truthy in Jinja. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub vars: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub threads: Option, + #[serde(default)] + pub full_refresh: bool, + /// Automatic in-job retry of the nodes a build failed on. + /// + /// dbt already confines a failure to its own subtree, and `dbt retry` + /// rebuilds exactly the failed and skipped set, so a transient warehouse + /// error costs those nodes rather than the whole project. Doing it inside + /// the same job is what keeps the state question out of it: the previous + /// attempt's `run_results.json` is still in the job directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_failed_nodes: Option, + /// Extra environment for the dbt process, for the project's own + /// `{{ env_var() }}` lookups and for engine flags such as + /// `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`. A `$var:` value is resolved to + /// that Windmill variable's value by the worker, so a password never has to + /// sit in the descriptor — which is versioned script content. + /// + /// This is the map to use for anything the GRAPH depends on — an + /// `env_var()` driving a schema, alias or `enabled` — because it applies at + /// deploy as well as at run. Script-level environment variables reach the + /// run only, so a graph parsed without them would disagree with what the + /// build writes. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, +} + +/// How many times, and how far apart, to retry a build's failed nodes. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct DbtNodeRetry { + /// Extra `dbt retry` attempts for the job, spent across whichever phases + /// fail — a model phase that uses them all leaves none for the tests. + pub attempts: u32, + /// Seconds between attempts. A transient warehouse error is usually a lock + /// or a restart, so a pause is the point. + #[serde(default)] + pub delay_seconds: u64, +} + +impl DbtNodeRetry { + /// Bounded because each attempt is a real dbt invocation inside a job that + /// already holds a worker slot, and the job's own deadline still applies. + pub const MAX_ATTEMPTS: u32 = 10; + + pub fn attempts(&self) -> u32 { + self.attempts.min(Self::MAX_ATTEMPTS) + } +} + +/// The dbt subcommands a run may ask for. Kept here so the signature and the +/// worker's validation cannot drift apart. +/// +/// An allowlist rather than a passthrough: the value becomes the dbt subcommand, +/// and running a script needs weaker permission than editing it, so an unchecked +/// arg would let a runner invoke `clean` or `source freshness` against the +/// descriptor's warehouse. +/// +/// `run` is absent because it covers models only: a project with seeds or +/// snapshots would build a subset of what its graph claims. Narrowing what a run +/// touches is `select`/`exclude`, which scope the graph too. Tests run as part +/// of `build`, or as the second phase of `test_behavior: after_all`. +/// +/// `show` writes nothing — it SELECTs from a model and returns rows. That is +/// only admissible because a dbt run no longer dispatches: while it did, any +/// successful job fired the script's whole deploy-time write set, so a command +/// that built none of them woke every consumer for relations nothing touched. +/// +/// `parse` reads no relation either: it resolves the project into a manifest and +/// stores the graph, which is what the editor's model panel is refreshed by. It +/// still needs a resolvable warehouse, because the profile is rendered before +/// any dbt invocation — a misconfigured project therefore fails a refresh the way +/// it would fail a run, which is the early feedback worth having. +pub const DBT_COMMANDS: &[&str] = &["build", "retry", "show", "parse"]; + +/// Rows a `show` returns unless the run asks for fewer. dbt enforces it, so the +/// bound is not us splicing a `LIMIT` into someone's SQL. +pub const DBT_SHOW_DEFAULT_LIMIT: u32 = 100; + +/// Hard ceiling on that limit. The worker buffers `dbt show`'s whole stdout to +/// read the rows out of it, so the argument decides how much memory a caller can +/// make it hold — and running a script needs only run permission. A preview is +/// for looking at a few rows; anything larger is a query, which is what a SQL +/// script is for. +pub const DBT_SHOW_MAX_LIMIT: u32 = 1_000; + +/// Whether the command only reads. Such a run publishes no graph, records no +/// materializations and runs no test phase — there is nothing it could have +/// changed. +pub fn is_read_only_command(command: &str) -> bool { + command == "show" +} + +/// The command a run uses when it does not name one. Public because the worker +/// must choose exactly what the run form's default advertises. +pub fn default_command(d: &DbtDescriptor) -> &'static str { + match d.test_behavior { + DbtTestBehavior::Build => "build", + // Also `build`, with the tests excluded rather than the command + // narrowed to `run`: `run` covers models only, so a selection that + // includes a seed or a snapshot would silently not build it and the + // models reading it would fail — or worse, read a stale table. + DbtTestBehavior::AfterAll | DbtTestBehavior::None => "build", + } +} + +impl DbtDescriptor { + pub fn engine(&self) -> DbtEngine { + self.engine.unwrap_or_default() + } +} + +/// Names a `{{ placeholder }}` may not take. `command` is the argument holding +/// the run's command block; the rest are the fields inside it, which the worker +/// spreads over the run's arguments to read them — a placeholder of the same +/// name would be shadowed there, and the script could never be run (`select` is +/// an array, and interpolating one into a string is not something any invocation +/// can satisfy). +pub const RESERVED_ARG_NAMES: &[&str] = &[ + "command", + "select", + "exclude", + "vars", + "full_refresh", + "dbt_command", + "dbt_retry_job", + "model", + "limit", +]; + +pub fn parse_dbt_descriptor(inner_content: &str) -> anyhow::Result { + let d = serde_yml::from_str::(inner_content) + .map_err(|e| anyhow::anyhow!("Failed to parse dbt descriptor: {e}"))?; + if let Some(name) = placeholders(&d) + .into_iter() + .find(|n| RESERVED_ARG_NAMES.contains(&n.as_str())) + { + return Err(anyhow::anyhow!( + "`{{{{ {name} }}}}` collides with the run argument `{name}` this runtime already \ + defines ({}); rename the placeholder", + RESERVED_ARG_NAMES.join(", ") + )); + } + Ok(d) +} + +/// The workspace warehouse a descriptor gets when it names none — spelled like +/// the default lake (`ducklake://main.orders`), so one workspace concept reads +/// the same across kinds. +pub const DBT_DEFAULT_WAREHOUSE: &str = "main"; + +/// The single argument holding the command and the overrides it takes. Its +/// variant IS the command, so a run cannot carry an override the command +/// ignores: `retry` rebuilds the failed run's nodes with the arguments that run +/// had, and `full_refresh` means nothing to a `show`. +pub const DBT_COMMAND_ARG: &str = "command"; + +/// The variant discriminator, which is the key Windmill's run form tags a +/// `oneOf` value with. +pub const DBT_COMMAND_LABEL: &str = "label"; + +fn list_arg(name: &str, default: &[String]) -> Arg { + Arg { + name: name.to_string(), + otyp: None, + typ: Typ::List(Box::new(Typ::Str(None))), + has_default: true, + default: Some(serde_json::json!(default)), + oidx: None, + otyp_inferred: false, + } +} + +/// An OVERRIDE map, so its default is empty rather than a copy of the +/// descriptor's vars. Seeding it with the descriptor would make the run form +/// post the raw `{{ placeholder }}` text back and clobber the value the worker +/// just interpolated for it. +fn vars_arg() -> Arg { + Arg { + name: "vars".to_string(), + otyp: None, + typ: Typ::Object(windmill_parser::ObjectType::new(None, Some(vec![]))), + has_default: true, + default: Some(serde_json::json!({})), + oidx: None, + otyp_inferred: false, + } +} + +/// What each command takes, in form order. +/// +/// `show` and `parse` are absent on purpose, for the same reason: each is a thing +/// to do to the project you are looking at rather than a job to fill a form in +/// for. `show` previews ONE model's rows, offered by the run page's graph and the +/// assets list where the tables are; `parse` refreshes the model graph, offered by +/// the dbt editor over the buffer being edited. The worker still accepts +/// `{label: show, model, limit}` and `{label: parse, vars}` from a flow, the CLI or +/// the API (`DBT_COMMANDS`, docs/dbt-runtime.md). +fn command_variants(d: &DbtDescriptor) -> Vec<(&'static str, Vec)> { + let selection = || { + vec![ + list_arg("select", &d.select), + list_arg("exclude", &d.exclude), + vars_arg(), + ] + }; + vec![ + ( + "build", + selection() + .into_iter() + .chain([Arg { + name: "full_refresh".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.full_refresh)), + oidx: None, + otyp_inferred: false, + }]) + .collect(), + ), + ( + "retry", + // The run it resumes, named rather than implied: only the latest + // failure of this script is kept, so "resume the last one" would + // silently aim somewhere else the moment another run failed. No + // default, so this variant requires it. + vec![Arg { + name: "dbt_retry_job".to_string(), + otyp: None, + typ: Typ::Str(None), + has_default: false, + default: None, + oidx: None, + otyp_inferred: false, + }], + ), + ] +} + +/// The command block a run gets when it overrides nothing: the descriptor's own +/// selection under the descriptor's default command, so an untouched run +/// reproduces it exactly. +fn default_command_value(d: &DbtDescriptor) -> serde_json::Value { + let label = default_command(d); + let mut obj = serde_json::Map::new(); + obj.insert(DBT_COMMAND_LABEL.to_string(), serde_json::json!(label)); + for arg in command_variants(d) + .into_iter() + .find(|(l, _)| *l == label) + .map(|(_, args)| args) + .unwrap_or_default() + { + if let Some(default) = arg.default { + obj.insert(arg.name, default); + } + } + serde_json::Value::Object(obj) +} + +/// Run-time arguments of a dbt script: the command block above, plus one +/// argument per `{{ placeholder }}` the descriptor interpolates. Placeholders +/// stay top-level — they are the project's own inputs, not a command's. +pub fn parse_dbt_sig(inner_content: &str) -> anyhow::Result { + let d = parse_dbt_descriptor(inner_content)?; + let mut args = vec![Arg { + name: DBT_COMMAND_ARG.to_string(), + otyp: None, + typ: Typ::Object(windmill_parser::ObjectType::new(None, Some(vec![]))), + has_default: true, + default: Some(default_command_value(&d)), + oidx: None, + otyp_inferred: false, + }]; + + for name in placeholders(&d) { + if args.iter().any(|a| a.name == name) { + continue; + } + args.push(Arg { + name, + otyp: None, + // Untyped, not `Str`: a placeholder standing alone in a var takes + // the argument's own JSON type, and declaring it a string makes the + // run form post `"false"` — truthy in Jinja — for a boolean. + typ: Typ::Unknown, + has_default: false, + default: None, + oidx: None, + otyp_inferred: false, + }); + } + + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + auto_kind: None, + has_preprocessor: None, + ..Default::default() + }) +} + +/// The run form's JSON schema for a dbt descriptor. +/// +/// Derived here rather than in the browser or the CLI: both infer a script's +/// schema client-side through `windmill-parser-wasm`, whose published package +/// has no dbt arm, so they leave the schema untouched. Without this a dbt +/// script deploys with an empty schema and its run form offers none of the +/// overrides — and an edited descriptor keeps the previous one's arguments. +/// The command is a `oneOf` rather than an enum beside the overrides it selects: +/// a variant carries exactly the arguments its command takes, so `dbt_retry_job` +/// is required where it means something and absent everywhere else, and no run +/// can submit a `full_refresh` to a command that ignores it. The run form renders +/// it as a toggle over the variants and tags the value with `label`. +pub fn dbt_arg_schema(inner_content: &str) -> anyhow::Result { + let d = parse_dbt_descriptor(inner_content)?; + let sig = parse_dbt_sig(inner_content)?; + let mut properties = serde_json::Map::new(); + let mut required: Vec = vec![]; + let mut order: Vec = vec![]; + + let variants: Vec = command_variants(&d) + .into_iter() + .map(|(label, args)| { + let mut props = serde_json::Map::new(); + let mut var_required: Vec = vec![]; + let mut var_order: Vec = vec![serde_json::json!(DBT_COMMAND_LABEL)]; + // The discriminator. Single-valued, so the form's toggle is what sets + // it and a submitted block cannot claim one command while carrying + // another's fields. + props.insert( + DBT_COMMAND_LABEL.to_string(), + serde_json::json!({"type": "string", "enum": [label]}), + ); + for arg in args { + if !arg.has_default { + var_required.push(serde_json::json!(arg.name)); + } + var_order.push(serde_json::json!(arg.name)); + props.insert(arg.name.clone(), property_of(&arg)); + } + serde_json::json!({ + "title": label, + "type": "object", + "properties": props, + "order": var_order, + "required": var_required, + }) + }) + .collect(); + + for arg in &sig.args { + let mut prop = property_of(arg); + if arg.name == DBT_COMMAND_ARG { + if let Some(obj) = prop.as_object_mut() { + obj.insert( + "oneOf".to_string(), + serde_json::Value::Array(variants.clone()), + ); + obj.insert( + "description".to_string(), + serde_json::json!( + "`build` runs the project. `retry` resumes a failed run, rebuilding only \ + its failed and skipped nodes with the arguments it ran with." + ), + ); + } + } + if !arg.has_default { + required.push(serde_json::json!(arg.name)); + } + order.push(serde_json::json!(arg.name)); + properties.insert(arg.name.clone(), prop); + } + Ok(serde_json::json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": properties, + "required": required, + "order": order, + })) +} + +/// One argument as a JSON-schema property, with the description the run form +/// shows under its label. +fn property_of(arg: &Arg) -> serde_json::Value { + let mut prop = match &arg.typ { + Typ::Bool => serde_json::json!({"type": "boolean"}), + // `limit` is an integer to the worker, which clamps it; without this the + // generated clients and the run form offer no numeric control. + Typ::Int => serde_json::json!({"type": "integer"}), + Typ::List(_) => serde_json::json!({"type": "array", "items": {"type": "string"}}), + Typ::Object(_) => serde_json::json!({"type": "object"}), + Typ::Str(Some(variants)) => serde_json::json!({"type": "string", "enum": variants}), + Typ::Str(None) => serde_json::json!({"type": "string"}), + // A placeholder takes the JSON type of whatever is passed, so it is + // deliberately left untyped rather than guessed at. + _ => serde_json::json!({}), + }; + if let Some(obj) = prop.as_object_mut() { + if let Some(default) = arg.default.as_ref() { + obj.insert("default".to_string(), default.clone()); + } + let description = match arg.name.as_str() { + "dbt_retry_job" => Some( + "The failed run to resume, by run id. Its failed and skipped nodes are rebuilt \ + with the arguments it ran with. Resuming from that run's page fills this in.", + ), + "select" => Some( + "dbt selection syntax, e.g. `tag:nightly`, `stg_orders+`, \ + `config.materialized:incremental`. Empty runs the descriptor's own selection.", + ), + "exclude" => Some("Nodes to leave out of the selection above, same syntax."), + "vars" => Some( + "dbt `--vars`, merged over the descriptor's. A var that changes which models \ + exist makes this run store its own graph rather than the deployed one.", + ), + "full_refresh" => Some("Rebuild incremental models from scratch instead of appending."), + "model" => Some( + "The model to preview, by name — `stg_orders`, or `my_package.stg_orders` when \ + two packages share a name. Any dbt selector resolving to ONE node works.", + ), + "limit" => Some("Rows the preview returns."), + // A placeholder the descriptor interpolates: its meaning is the + // project's, so there is nothing generic to say about it. + _ => None, + }; + if let Some(d) = description { + obj.insert("description".to_string(), serde_json::json!(d)); + } + } + prop +} + +/// `{{ name }}` placeholders in the interpolated descriptor fields, in a stable +/// order. Must stay in sync with the fields the worker actually interpolates. +fn placeholders(d: &DbtDescriptor) -> Vec { + let mut out: Vec = vec![]; + let mut push_from = |s: &str| { + for caps in PLACEHOLDER_RE.captures_iter(s) { + let name = caps[1].to_string(); + if !out.contains(&name) { + out.push(name); + } + } + }; + for v in d.vars.values() { + for leaf in string_leaves(v) { + push_from(leaf); + } + } + out +} + +/// Every string inside a var's value, at any depth — the only places a +/// `{{ arg }}` placeholder can appear. +pub fn string_leaves(v: &serde_json::Value) -> Vec<&str> { + match v { + serde_json::Value::String(s) => vec![s.as_str()], + serde_json::Value::Array(a) => a.iter().flat_map(string_leaves).collect(), + serde_json::Value::Object(o) => o.values().flat_map(string_leaves).collect(), + _ => vec![], + } +} + +lazy_static::lazy_static! { + /// Same spelling as the Ansible executor's `interpolate_template`, which is + /// what actually performs the substitution at run time. + static ref PLACEHOLDER_RE: regex::Regex = + regex::Regex::new(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}").unwrap(); +} + +#[cfg(test)] +mod tests { + use super::*; + + const DESCRIPTOR: &str = r#" +engine: dbt-core-2x +profile: + warehouse: main + target: prod +select: ["tag:nightly+"] +test_behavior: after_all +vars: + run_date: "{{ day }}" + strict: false +threads: 8 +full_refresh: true +"#; + + // A descriptor drives warehouse writes, so a field it does not recognise is + // an error rather than a default. `selcet:` would otherwise leave `select` + // empty and build the entire project; a misspelled `target` would silently + // fall back to the profile's own default. + #[test] + fn an_unknown_descriptor_field_is_refused() { + let err = parse_dbt_descriptor("profile:\n warehouse: main\nselcet: [a]\n") + .unwrap_err() + .to_string(); + assert!(err.contains("selcet"), "{err}"); + let err = parse_dbt_descriptor("profile:\n warehouse: main\n targt: prod\n") + .unwrap_err() + .to_string(); + assert!(err.contains("targt"), "{err}"); + } + + // A placeholder that takes a run argument's name would be shadowed by it, + // leaving a descriptor no invocation can satisfy: `select` is an array and + // the interpolation needs a scalar. Refused at parse, so the deploy says so + // rather than the script becoming unrunnable after it. + #[test] + fn a_placeholder_may_not_take_a_run_argument_name() { + for name in RESERVED_ARG_NAMES { + let d = format!("profile:\n warehouse: main\nvars:\n v: \"{{{{ {name} }}}}\"\n"); + let err = parse_dbt_descriptor(&d).unwrap_err().to_string(); + assert!(err.contains(name), "{name}: {err}"); + } + // A name of its own is fine. + assert!( + parse_dbt_descriptor("profile:\n warehouse: main\nvars:\n v: \"{{ day }}\"\n") + .is_ok() + ); + } + + // The run form is built from this schema, and nothing else can build it: + // the browser and the CLI both infer through a wasm parser that has no dbt + // arm, so a missing property here is an override the user cannot reach. + // The variants are what make the form show a command's own fields and only + // those, so each is asserted by the arguments it carries. + #[test] + fn the_schema_carries_every_run_override_and_placeholder() { + let schema = dbt_arg_schema(DESCRIPTOR).unwrap(); + let props = schema["properties"].as_object().unwrap(); + let variants = props[DBT_COMMAND_ARG]["oneOf"].as_array().unwrap(); + let of = |label: &str| { + let v = variants + .iter() + .find(|v| v["title"] == label) + .unwrap_or_else(|| panic!("no `{label}` variant: {schema}")); + let mut names: Vec = v["properties"] + .as_object() + .unwrap() + .keys() + .filter(|k| k.as_str() != DBT_COMMAND_LABEL) + .cloned() + .collect(); + names.sort(); + (v.clone(), names) + }; + + let (build, build_args) = of("build"); + assert_eq!(build_args, ["exclude", "full_refresh", "select", "vars"]); + assert_eq!(build["properties"]["full_refresh"]["type"], "boolean"); + // Defaults come from the descriptor, so an untouched run reproduces it. + assert_eq!( + build["properties"]["select"]["default"], + serde_json::json!(["tag:nightly+"]) + ); + // The discriminator takes one value per variant: the toggle sets it, and + // a block cannot claim one command while carrying another's fields. + assert_eq!( + build["properties"][DBT_COMMAND_LABEL]["enum"], + serde_json::json!(["build"]) + ); + + let (retry, retry_args) = of("retry"); + assert_eq!(retry_args, ["dbt_retry_job"]); + // Required where it means something, rather than required everywhere and + // hidden: a retry names the run it resumes. + assert_eq!(retry["required"], serde_json::json!(["dbt_retry_job"])); + + // Neither `show` nor `parse` is a form variant: previewing one model's + // rows and refreshing the model graph are both things you do to the + // project in front of you, and the graph, the assets list and the dbt + // editor are where those live. The worker still accepts both + // programmatically, which is what makes them scriptable. + for hidden in ["show", "parse"] { + assert!( + !variants.iter().any(|v| v["title"] == hidden), + "{hidden} must not be offered in the run form: {schema}" + ); + assert!( + DBT_COMMANDS.contains(&hidden), + "but the worker still takes {hidden}" + ); + } + + // Every `{{ placeholder }}` the descriptor interpolates is an argument a + // run must supply. The command block is not one: it defaults to the + // descriptor's own selection under the descriptor's command. + assert_eq!(schema["required"], serde_json::json!(["day"])); + assert_eq!(schema["order"], serde_json::json!([DBT_COMMAND_ARG, "day"])); + } + + #[test] + fn parses_descriptor() { + let d = parse_dbt_descriptor(DESCRIPTOR).unwrap(); + assert_eq!(d.engine(), DbtEngine::DbtCore2x); + assert_eq!(d.profile.target.as_deref(), Some("prod")); + assert_eq!(d.select, vec!["tag:nightly+"]); + assert_eq!(d.threads, Some(8)); + assert!(d.full_refresh); + // dbt vars keep their YAML type: a `false` coerced to "false" is truthy + // in Jinja and would silently invert the condition it gates. + assert_eq!(d.vars["strict"], serde_json::json!(false)); + assert_eq!(d.vars["run_date"], serde_json::json!("{{ day }}")); + } + + #[test] + fn an_empty_descriptor_defaults_to_the_bundled_engine() { + let d = parse_dbt_descriptor("").unwrap(); + assert_eq!(d.engine(), DbtEngine::DbtCore1x); + } + + // The ORDER is asserted, not just the set: the schema's `order` is built from + // this vec and the run form follows it, so the command block leading is what + // puts the choice of what the run does above the project's own inputs. + #[test] + fn signature_exposes_the_command_block_and_placeholders() { + let sig = parse_dbt_sig(DESCRIPTOR).unwrap(); + let names: Vec<&str> = sig.args.iter().map(|a| a.name.as_str()).collect(); + assert_eq!(names, vec![DBT_COMMAND_ARG, "day"]); + // An untouched run reproduces the descriptor: its default is the whole + // block — the descriptor's command, carrying the descriptor's selection. + let cmd = sig.args.iter().find(|a| a.name == DBT_COMMAND_ARG).unwrap(); + let default = cmd.default.clone().unwrap(); + // `build`, not `run`: `run` covers models only, so a selection naming a + // seed or a snapshot would silently not build it. + assert_eq!(default[DBT_COMMAND_LABEL], serde_json::json!("build")); + assert_eq!(default["select"], serde_json::json!(["tag:nightly+"])); + assert_eq!(default["full_refresh"], serde_json::json!(true)); + // `vars` is the exception: it overrides, so its default must be empty. + // Seeded with the descriptor, the run form would post `{{ day }}` back + // and overwrite the value the worker interpolated for it. + assert_eq!(default["vars"], serde_json::json!({})); + // Placeholders are required (the descriptor names no value for them) + // and untyped, so a `{{ }}` var can carry a boolean or a number rather + // than the string "false", which Jinja treats as truthy. + let day = sig.args.iter().find(|a| a.name == "day").unwrap(); + assert!(!day.has_default); + assert_eq!(day.typ, Typ::Unknown); + } +} diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index f2a1a2b824..e025849769 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -9,6 +9,13 @@ use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; pub mod asset_parser; pub use asset_parser::parse_assets; +pub mod dbt; +pub use dbt::{ + dbt_arg_schema, default_command as default_dbt_command, parse_dbt_descriptor, parse_dbt_sig, + DbtDescriptor, DbtEngine, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, DBT_COMMAND_LABEL, + DBT_DEFAULT_WAREHOUSE, +}; + pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result { let docs = YamlLoader::load_from_str(inner_content) .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?; diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index de35ccdd6e..843a83a310 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -1,4 +1,5 @@ use serde::Serialize; +use std::borrow::Cow; use std::collections::BTreeMap; // Token recognized inside declared asset URIs that the runtime substitutes @@ -28,6 +29,11 @@ pub enum AssetKind { Ducklake, DataTable, Volume, + /// A warehouse relation a dbt project builds or reads, + /// `dbt:////`, the warehouse named as the + /// workspace configures it. The scheme names the producer, the path stays + /// the relation — see `windmill_types::AssetKind::Dbt`. + Dbt, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -714,21 +720,32 @@ pub fn asset_was_used(assets: &Vec, (kind, path): (AssetKind, }) } -pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(AssetKind, &str)> { +/// Split an asset URI into `(kind, path)`. The single point where user-written +/// asset URIs become graph keys, and therefore where `dbt://` paths get +/// canonicalized (see `canonicalize_table_asset_path`) — every other kind's +/// suffix is kept verbatim and stays borrowed. +pub fn parse_asset_syntax( + s: &str, + enable_default_syntax: bool, +) -> Option<(AssetKind, Cow<'_, str>)> { if enable_default_syntax && s == "datatable" { - return Some((AssetKind::DataTable, "main")); + return Some((AssetKind::DataTable, Cow::Borrowed("main"))); } else if enable_default_syntax && s == "ducklake" { - return Some((AssetKind::Ducklake, "main")); + return Some((AssetKind::Ducklake, Cow::Borrowed("main"))); } for (prefix, kind) in ASSET_KINDS.iter() { if s.starts_with(prefix) { + let suffix = &s[prefix.len()..]; + if *kind == AssetKind::Dbt { + return Some((*kind, Cow::Owned(canonicalize_table_asset_path(suffix)))); + } // The suffix is kept verbatim. For S3 the path encodes the storage: // `s3:///`, with an EMPTY storage segment for the // workspace default — so `s3:///key` yields `/key` (leading slash // significant, default storage) while `s3://secondary/key` yields // `secondary/key`. Stripping leading slashes here would conflate a // default-storage object with a named-storage one. - return Some((*kind, &s[prefix.len()..])); + return Some((*kind, Cow::Borrowed(suffix))); } } None @@ -741,8 +758,123 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("ducklake://", AssetKind::Ducklake), ("datatable://", AssetKind::DataTable), ("volume://", AssetKind::Volume), + ("dbt://", AssetKind::Dbt), ]; +/// Canonical spelling of a `dbt://` path, `//`. +/// +/// The whole point of keying warehouse relations on the relation rather than on +/// the producing tool is that a dbt mart and a native script reading the same +/// table land on one graph node. That only holds if both sides spell the key +/// identically, and they will not by default: dbt's `manifest.json` gives +/// `relation_name` pre-quoted (`"db"."Schema"."Tbl"`) while an annotation is +/// written by hand, and the warehouses disagree on case (Snowflake folds +/// unquoted identifiers up, Postgres folds them down, DuckDB compares them +/// case-insensitively). Two spellings of one table produce two nodes, no edge, +/// and nothing looks broken in isolation — so the rule is applied once, here, +/// on every path that becomes a `dbt://` asset. +/// +/// The rule: strip the quote characters the warehouses use (`"`, backtick, +/// `[`/`]`) from the schema and name, then ASCII-lowercase them. This matches +/// the case-insensitive identifier comparison the DuckDB paths already use +/// (`schema_contracts::CapturedSchema::find`). The warehouse-name prefix is +/// spelled as the workspace configures it, which is case-sensitive, and is left +/// untouched. +/// +/// Consequence to accept: a relation deliberately created under a quoted +/// mixed-case identifier collides with its lowercase spelling. That is rarer +/// than the case-fold mismatch it prevents, and it errs toward unifying nodes +/// rather than splitting them. +pub fn canonicalize_table_asset_path(path: &str) -> String { + let Some((resource, rest)) = path.rsplit_once('/').and_then(|(head, name)| { + head.rsplit_once('/') + .map(|(resource, schema)| (resource, (schema, name))) + }) else { + // Fewer than three segments: not a well-formed relation path. Leave it + // alone so the malformed value stays visible instead of being reshaped + // into something that looks valid. + return path.to_string(); + }; + let (schema, name) = rest; + format!( + "{}/{}/{}", + resource, + unquote_schema_segment(schema).to_ascii_lowercase(), + unquote_identifier(name).to_ascii_lowercase() + ) +} + +/// A doubled delimiter inside a quoted identifier is that delimiter, literally — +/// the same rule the worker's `split_relation` applies to `relation_name`. Both +/// have to decode it or one spelling of a table becomes two graph nodes: the dbt +/// model under `sales"east`, its native consumer under `sales""east`. +fn unquote_identifier(s: &str) -> String { + let s = s.trim(); + for (open, close) in [('"', '"'), ('`', '`'), ('[', ']')] { + if s.len() >= 2 && s.starts_with(open) && s.ends_with(close) { + let inner = &s[1..s.len() - 1]; + return inner.replace(&format!("{close}{close}"), &close.to_string()); + } + } + s.to_string() +} + +/// Unquote a schema segment, which carries a `.` pair when a +/// relation overrode its database. Each half is separately quotable, so +/// stripping only the outer pair leaves `"Archive"."Sales"` as +/// `Archive"."Sales` — a different key from the ingest's `archive.sales`, which +/// silently splits the model and its native consumer into two nodes. +/// +/// The halves go through `unquote_identifier`, which requires the quote at both +/// ends, because this function sees both a warehouse SPELLING (from a `dbt://` +/// annotation) and an already-DECODED identifier (from `table_asset_path`, whose +/// callers ran `split_relation` or read the manifest) and cannot tell them apart. +/// A lone `"` in a decoded name — `sa"les` — must survive: treated as opening a +/// quote it would be dropped, filing the ingest's `sa"les` under `sales` while +/// the annotation's `"sa""les"` decodes to `sa"les`, which is the split this +/// canonicalization exists to prevent. +fn unquote_schema_segment(s: &str) -> String { + let s = s.trim(); + let mut parts: Vec<&str> = Vec::new(); + let mut start = 0usize; + let mut quote: Option = None; + let bytes: Vec<(usize, char)> = s.char_indices().collect(); + let mut k = 0usize; + while k < bytes.len() { + let (i, c) = bytes[k]; + match quote { + Some(q) => { + let close = if q == '[' { ']' } else { q }; + if c == close { + // Doubled: a literal delimiter, so the quote stays open. + if bytes.get(k + 1).map(|(_, n)| *n) == Some(close) { + k += 1; + } else { + quote = None; + } + } + } + // Only a quote that OPENS the part can be a delimiter; one in the + // middle of a decoded identifier is part of the name. + None if (c == '"' || c == '`' || c == '[') && i == start => quote = Some(c), + // Only an UNQUOTED period separates the database from the schema; + // one inside an identifier is part of the name. + None if c == '.' => { + parts.push(&s[start..i]); + start = i + c.len_utf8(); + } + None => {} + } + k += 1; + } + parts.push(&s[start..]); + parts + .into_iter() + .map(unquote_identifier) + .collect::>() + .join(".") +} + // Tokenize a `key=value [key="quoted value"] ...` option string. Bare // values run until the next whitespace; quoted values consume until the // matching quote. Malformed pairs (missing `=` or empty key) are skipped @@ -1555,11 +1687,11 @@ mod pipeline_annotation_tests { // objects and must never collapse to one identity. assert_eq!( parse_asset_syntax("s3:///exports/x", false), - Some((AssetKind::S3Object, "/exports/x")) + Some((AssetKind::S3Object, Cow::Borrowed("/exports/x"))) ); assert_eq!( parse_asset_syntax("s3://exports/x", false), - Some((AssetKind::S3Object, "exports/x")) + Some((AssetKind::S3Object, Cow::Borrowed("exports/x"))) ); assert_ne!( parse_asset_syntax("s3:///exports/x", false), @@ -1569,28 +1701,181 @@ mod pipeline_annotation_tests { // The `// on` trigger annotation goes through the same function. assert_eq!( parse_asset_syntax("s3:///exports/x", true), - Some((AssetKind::S3Object, "/exports/x")) + Some((AssetKind::S3Object, Cow::Borrowed("/exports/x"))) ); assert_eq!( parse_asset_syntax("s3://secondary_storage/path/to/file.csv", false), - Some((AssetKind::S3Object, "secondary_storage/path/to/file.csv")) + Some(( + AssetKind::S3Object, + Cow::Borrowed("secondary_storage/path/to/file.csv") + )) ); // Hive-partition keys are preserved verbatim. assert_eq!( parse_asset_syntax("s3:///t/year=2024/month=01/f.parquet", false), - Some((AssetKind::S3Object, "/t/year=2024/month=01/f.parquet")) + Some(( + AssetKind::S3Object, + Cow::Borrowed("/t/year=2024/month=01/f.parquet") + )) ); // Non-S3 kinds also keep their suffix verbatim. assert_eq!( parse_asset_syntax("res://f/foo", false), - Some((AssetKind::Resource, "f/foo")) + Some((AssetKind::Resource, Cow::Borrowed("f/foo"))) ); assert_eq!( parse_asset_syntax("ducklake://analytics/orders", false), - Some((AssetKind::Ducklake, "analytics/orders")) + Some((AssetKind::Ducklake, Cow::Borrowed("analytics/orders"))) + ); + } + + // A dbt mart and a native script reading the same warehouse table must land + // on ONE graph node. They only do if every spelling of the relation + // canonicalizes identically — dbt's manifest gives it pre-quoted, the + // annotation is hand-written, and the warehouses fold case in opposite + // directions. A regression here is invisible: both nodes still render, they + // just stop being the same node and the cross-boundary cascade never fires. + #[test] + fn table_paths_from_every_spelling_canonicalize_to_one_key() { + let canonical = Some(( + AssetKind::Dbt, + Cow::Owned("main/analytics/orders".into()), + )); + for spelling in [ + // Hand-written annotation. + "dbt://main/analytics/orders", + // dbt manifest `relation_name`, quoted (database segment already + // dropped by the ingest, which keys on the warehouse name instead). + "dbt://main/\"analytics\"/\"orders\"", + // Snowflake folds unquoted identifiers up, Postgres folds down. + "dbt://main/ANALYTICS/ORDERS", + "dbt://main/Analytics/Orders", + // BigQuery / Databricks backticks, SQL Server brackets. + "dbt://main/`analytics`/`orders`", + "dbt://main/[Analytics]/[Orders]", + ] { + assert_eq!( + parse_asset_syntax(spelling, false), + canonical, + "spelling {spelling} did not canonicalize" + ); + } + } + + // A relation that overrode its database carries `.` in + // one segment, and each half can be quoted independently. Stripping only + // the outer pair leaves a key the manifest ingest never produces, so the + // model and a native script reading it become separate nodes. + #[test] + fn qualified_schema_segments_unquote_each_half() { + let canonical = Some(( + AssetKind::Dbt, + Cow::Owned("main/archive.sales/orders".into()), + )); + for spelling in [ + "dbt://main/archive.sales/orders", + "dbt://main/\"Archive\".\"Sales\"/\"Orders\"", + "dbt://main/`Archive`.`Sales`/`Orders`", + "dbt://main/[Archive].[Sales]/[Orders]", + ] { + assert_eq!( + parse_asset_syntax(spelling, false), + canonical, + "spelling {spelling} did not canonicalize" + ); + } + // A period INSIDE one quoted identifier is part of the name, not a + // database qualifier. + assert_eq!( + parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), + Some(( + AssetKind::Dbt, + Cow::Owned("main/sales.v2/orders".into()) + )) + ); + } + + // Every dialect here escapes its own delimiter by doubling it, and the + // worker's `split_relation` decodes it — so a canonicalizer that did not + // would file the dbt model under `sales"east` and the native script reading + // it under `sales""east`, which is the split this key exists to prevent. + #[test] + fn a_doubled_delimiter_canonicalizes_like_the_relation_it_names() { + for (spelling, expected) in [ + ( + "dbt://main/\"sales\"\"east\"/\"orders\"", + "main/sales\"east/orders", + ), + ( + "dbt://main/analytics/\"order\"\"s\"", + "main/analytics/order\"s", + ), + ( + "dbt://main/`da``ta`/`orders`", + "main/da`ta/orders", + ), + ( + "dbt://main/[my]]schema]/[orders]", + "main/my]schema/orders", + ), + // And in one half of a database-qualified segment. + ( + "dbt://main/\"arch\"\"ive\".\"sales\"/orders", + "main/arch\"ive.sales/orders", + ), + ] { + assert_eq!( + parse_asset_syntax(spelling, false), + Some((AssetKind::Dbt, Cow::Owned(expected.into()))), + "spelling {spelling} did not canonicalize" + ); + } + // The DECODED form has to land on the same key, in both halves: this + // function sees the warehouse's spelling from an annotation and an + // already-decoded identifier from the ingest, and cannot tell them + // apart. A lone delimiter treated as opening a quote would be dropped — + // `sa"les` filed as `sales` — and the two derivations would split. + for (decoded, spelled) in [ + ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), + ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/arch\"ive.sales/orders", + "dbt://main/\"arch\"\"ive\".\"sales\"/orders", + ), + ] { + assert_eq!( + parse_asset_syntax(decoded, false), + parse_asset_syntax(spelled, false), + "the decoded and quoted spellings of {decoded} disagree" + ); + } + } + + #[test] + fn table_canonicalization_leaves_the_warehouse_name_case_alone() { + // The warehouse-name prefix is spelled as the workspace configures it + // and is case-sensitive; only the two identifier segments are folded. + assert_eq!( + parse_asset_syntax("dbt://MyWarehouse/Sales/Orders", false), + Some(( + AssetKind::Dbt, + Cow::Owned("MyWarehouse/sales/orders".into()) + )) + ); + // A database-qualified schema keeps its own case rules: the whole + // segment folds, dot included. + assert_eq!( + parse_asset_syntax("dbt://main/PROD.Sales/Orders", false), + Some((AssetKind::Dbt, Cow::Owned("main/prod.sales/orders".into()))) + ); + // Too few segments to be a relation: left alone rather than reshaped + // into something that looks well-formed. + assert_eq!( + parse_asset_syntax("dbt://Orders", false), + Some((AssetKind::Dbt, Cow::Owned("Orders".into()))) ); } diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 1936f7ff70..9493a78606 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -110,6 +110,7 @@ fn kind_str(k: AssetKind) -> &'static str { AssetKind::Ducklake => "ducklake", AssetKind::DataTable => "datatable", AssetKind::Volume => "volume", + AssetKind::Dbt => "dbt", } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 969fd4e94d..51febad6f0 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -932,46 +932,62 @@ async fn windmill_main() -> anyhow::Result<()> { } // Lower the worker's oom_score_adj so the OOM killer strongly prefers killing - // job subprocesses (oom_score_adj=1000) over the worker itself. + // job subprocesses (oom_score_adj=JOB_OOM_SCORE_ADJ) over the worker itself. // Kubernetes sets it high for burstable QoS (e.g. 937), leaving a tiny gap vs jobs. // Requires CAP_SYS_RESOURCE to lower it; if missing, we just warn. #[cfg(any(target_os = "linux"))] - match std::fs::read_to_string("/proc/self/oom_score_adj") { - Ok(current) => { - let current = current.trim().to_string(); - let current_val = match current.parse::() { - Ok(v) => v, - Err(e) => { - tracing::warn!("Could not parse oom_score_adj '{current}': {e}"); - 0 - } - }; - if current_val > 0 { - match std::fs::write("/proc/self/oom_score_adj", "0") { - Ok(_) => { - tracing::info!( - "Lowered worker oom_score_adj from {current} to 0 \ - (jobs get 1000, gap=1000)" - ); + { + // Badness is (memory used, in permille of host RAM) + oom_score_adj, so the gap + // must exceed the worker's own footprint in permille to actually steer the kill. + // 100 covers a worker holding up to ~10% of host RAM. + const MIN_OOM_SCORE_GAP: i32 = 100; + + let job_adj = *windmill_common::worker::JOB_OOM_SCORE_ADJ; + match std::fs::read_to_string("/proc/self/oom_score_adj") { + Ok(current) => { + let current = current.trim().to_string(); + match current.parse::() { + Ok(mut worker_adj) => { + if worker_adj > 0 { + match std::fs::write("/proc/self/oom_score_adj", "0") { + Ok(_) => { + tracing::info!( + "Lowered worker oom_score_adj from {worker_adj} to 0" + ); + worker_adj = 0; + } + Err(e) => { + tracing::warn!( + "Could not lower worker oom_score_adj from {worker_adj} to 0: {e}. \ + Add CAP_SYS_RESOURCE to the container to fix this" + ); + } + } + } + let gap = job_adj - worker_adj; + if gap >= MIN_OOM_SCORE_GAP { + tracing::info!( + "Worker oom_score_adj={worker_adj}, jobs get {job_adj} (gap={gap})" + ); + } else { + tracing::warn!( + "Worker oom_score_adj={worker_adj}, jobs get {job_adj} (gap={gap}): \ + too small to reliably steer the OOM killer to the job. \ + Raise JOB_OOM_SCORE_ADJ or lower the worker's own score" + ); + } } Err(e) => { tracing::warn!( - "Could not lower worker oom_score_adj from {current} to 0: {e}. \ - Gap to jobs is only {} — OOM killer may target the worker instead. \ - Add CAP_SYS_RESOURCE to the container to fix this", - 1000 - current_val + "Could not parse worker oom_score_adj '{current}': {e}. \ + Cannot tell whether jobs (oom_score_adj={job_adj}) outrank the worker" ); } } - } else { - tracing::info!( - "Worker oom_score_adj={current} (jobs get 1000, gap={})", - 1000 - current_val - ); } - } - Err(e) => { - tracing::warn!("Could not read worker oom_score_adj: {e}"); + Err(e) => { + tracing::warn!("Could not read worker oom_score_adj: {e}"); + } } } } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index ccff5c2c7b..336956415e 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -46,7 +46,7 @@ use windmill_common::otel_oss::{ use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::APP_WORKSPACED_ROUTE, - auth::create_token_for_owner, + auth::{create_token_for_owner, ephemeral_script_token_label}, ee_oss::CriticalErrorChannel, email_oss::send_email_if_possible, error, @@ -4662,13 +4662,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n continue; } if let Some(job) = job.unwrap() { - let label = if job.permissioned_as != format!("u/{}", job.created_by) - && job.permissioned_as != job.created_by - { - format!("ephemeral-script-end-user-{}", job.created_by) - } else { - "ephemeral-script".to_string() - }; + let label = ephemeral_script_token_label(&job.permissioned_as, &job.created_by); let token = create_token_for_owner( &db, &job.workspace_id, diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index b47a6556cd..44b62ca51e 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -3,7 +3,7 @@ ## ENUMs action_kind: create, update, delete, execute asset_access_type: r, w, rw -asset_kind: s3object, resource, variable, ducklake, datatable +asset_kind: s3object, resource, variable, ducklake, datatable, table asset_usage_kind: script, flow, job authentication_method: none, windmill, api_key, basic_http, custom_script, signature autoscaling_event_type: full_scaleout, scalein, scaleout @@ -19,6 +19,7 @@ job_status: success, failure, canceled, skipped job_trigger_kind: webhook, http, websocket, kafka, email, nats, schedule, app, ui, postgres, sqs, gcp, mqtt, nextcloud, ci_test log_mode: standalone, server, worker, agent, indexer, mcp login_type: password, github +materialization_status: running, materialized, failed message_type: user, assistant, tool metric_kind: scalar_int, scalar_float, timeseries_int, timeseries_float mqtt_client_version: v3, v5 @@ -26,7 +27,7 @@ native_trigger_service: nextcloud request_type: sync, async, sync_sse runnable_type: ScriptHash, ScriptPath, FlowPath script_kind: script, trigger, failure, command, approval, preprocessor -script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby, rlang +script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby, rlang, dbt trigger_kind: webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp, default_email, nextcloud trigger_mode: enabled, disabled, suspended workspace_key_kind: cloud @@ -65,6 +66,16 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) +dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) + FK: (workspace_id) -> workspace(id) +dbt_run_state: workspace_id(char), script_path(char), permissioned_as(char), identity(text), args(jsonb), run_results(text), job_id(uuid), retryable(bool), updated_at(ts) + FK: (workspace_id) -> workspace(id) debounce_key: key(char), job_id(uuid), previous_job_id(uuid), first_started_at(ts), debounced_times(int) debounce_stale_data: job_id(uuid), to_relock(text[]) debouncing_settings: hash(bigint), debounce_key(char), debounce_delay_s(int), max_total_debouncing_time(int), max_total_debounces_amount(int), debounce_args_to_accumulate(text[]) @@ -76,7 +87,7 @@ draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at( email_to_igroup: email(char), igroup(char) email_trigger: path(char), local_part(char), workspaced_local_part(bool), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) favorite: usr(char), workspace_id(char), path(char), favorite_kind(favorite_kind) -flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of_email(text), lock_error_logs(text), labels(text[]) +flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of(varchar), on_behalf_of_email(text), lock_error_logs(text), labels(text[]) FK: (workspace_id) -> workspace(id) flow_conversation: id(uuid), workspace_id(char), flow_path(char), title(char), created_at(ts), updated_at(ts), created_by(char) FK: (workspace_id) -> workspace(id) @@ -153,7 +164,7 @@ resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), appro runnable_settings: hash(bigint), debouncing_settings(bigint), concurrency_settings(bigint) schedule: workspace_id(char), path(char), edited_by(char), edited_at(ts), schedule(char), enabled(bool), script_path(char), args(jsonb), extra_perms(jsonb), is_flow(bool), email(char), error(text), timezone(char), on_failure(char), on_recovery(char), on_failure_times(int), on_failure_exact(bool), on_failure_extra_args(jsonb), on_recovery_times(int), on_recovery_extra_args(jsonb), ws_error_handler_muted(bool), retry(jsonb), summary(char), no_flow_overlap(bool), tag(char), paused_until(ts), on_success(char), on_success_extra_args(jsonb), cron_version(text), description(text), dynamic_skip(char), labels(text[]) FK: (workspace_id) -> workspace(id) -script: workspace_id(char), hash(bigint), path(char), parent_hashes(bigint[]), summary(text), description(text), content(text), created_by(char), created_at(ts), archived(bool), schema(json), deleted(bool), is_template(bool), extra_perms(jsonb), lock(text), lock_error_logs(text), language(script_lang), kind(script_kind), tag(char), draft_only(bool), envs(char), concurrent_limit(int), concurrency_time_window_s(int), cache_ttl(int), dedicated_worker(bool), ws_error_handler_muted(bool), priority(smallint), timeout(int), delete_after_use(bool), restart_unless_cancelled(bool), concurrency_key(char), visible_to_runner_only(bool), auto_kind(varchar), codebase(char), has_preprocessor(bool), on_behalf_of_email(text), schema_validation(bool), assets(jsonb), debounce_key(char), debounce_delay_s(int), cache_ignore_s3_path(bool), runnable_settings_handle(bigint), labels(text[]) +script: workspace_id(char), hash(bigint), path(char), parent_hashes(bigint[]), summary(text), description(text), content(text), created_by(char), created_at(ts), archived(bool), schema(json), deleted(bool), is_template(bool), extra_perms(jsonb), lock(text), lock_error_logs(text), language(script_lang), kind(script_kind), tag(char), draft_only(bool), envs(char), concurrent_limit(int), concurrency_time_window_s(int), cache_ttl(int), dedicated_worker(bool), ws_error_handler_muted(bool), priority(smallint), timeout(int), delete_after_use(bool), restart_unless_cancelled(bool), concurrency_key(char), visible_to_runner_only(bool), auto_kind(varchar), codebase(char), has_preprocessor(bool), schema_validation(bool), assets(jsonb), debounce_key(char), debounce_delay_s(int), cache_ignore_s3_path(bool), runnable_settings_handle(bigint), labels(text[]), on_behalf_of(varchar), on_behalf_of_email(text) FK: (workspace_id) -> workspace(id) skip_workspace_diff_tally: workspace_id(char), added_at(ts) sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attributes(text[]), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), error(text), server_id(char), last_server_ping(ts), aws_auth_resource_type(aws_auth_resource_type), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) @@ -204,7 +215,7 @@ workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_gr FK: (workspace_id) -> workspace(id) workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/tests/app_anonymous_execution_mode.rs b/backend/tests/app_anonymous_execution_mode.rs index a40d880f1b..286581c042 100644 --- a/backend/tests/app_anonymous_execution_mode.rs +++ b/backend/tests/app_anonymous_execution_mode.rs @@ -282,8 +282,8 @@ async fn test_restrict_anonymous_app_deployment_rule(db: Pool) -> anyh async fn seed_script(db: &Pool, path: &str, content: &str) -> anyhow::Result<()> { sqlx::query( r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, - created_by, on_behalf_of_email, language, tag, lock) - VALUES ('test-workspace', hashtext($2)::bigint, $1, '', '', $2, 'test-user', 'test@windmill.dev', + created_by, on_behalf_of, language, tag, lock) + VALUES ('test-workspace', hashtext($2)::bigint, $1, '', '', $2, 'test-user', 'u/test-user', 'deno'::script_lang, 'deno', '')"#, ) .bind(path) diff --git a/backend/tests/app_preview_auth.rs b/backend/tests/app_preview_auth.rs index 2a69004a7b..cd84e6f278 100644 --- a/backend/tests/app_preview_auth.rs +++ b/backend/tests/app_preview_auth.rs @@ -22,7 +22,9 @@ //! - run mode against a deployed Viewer app rejects caller-supplied inline //! `raw_code` whose sha is not publisher-pinned (CVE-2026-22683 residual: //! the Viewer default-triggerable fallback let any caller / an operator run -//! arbitrary code as themselves, bypassing the content-hash pin). +//! arbitrary code as themselves, bypassing the content-hash pin), and +//! - a path-qualified `apps:run|write:` token reaches only that app +//! (the route layer is resource-blind, so the handler must path-check). use serde_json::json; use sqlx::{Pool, Postgres}; @@ -344,5 +346,47 @@ async fn test_app_preview_authorization(db: Pool) -> anyhow::Result<() "rejection must be the content-hash pin (unpinned app_script sha), got: {body}" ); + // 12. Path confinement: the scope picker mints `apps:run:` / + // `apps:write:`, but the route layer matches domain + action only, so the + // handler is the only place the path is enforced. A token scoped to `vapp` must + // not execute another app's components — those run under that app's identity. + for token in ["APPS_RUN_VAPP_TOKEN", "APPS_WRITE_VAPP_TOKEN"] { + let resp = authed(client().post(format!("{base}/u/test-user/private")), token) + .json(&json!({ "args": {}, "component": "comp", "path": "script/u/test-user/private" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 403, + "{token} must not execute an app outside its scope path (got {status}): {body}" + ); + assert!( + body.contains("apps:run:u/test-user/private"), + "rejection must be the app path scope gate, got: {body}" + ); + } + + // 13. ...and must not over-block the app it IS scoped to: both tokens clear the + // scope gate for `vapp` and reach the policy (which then rejects the unpinned + // code, as in step 9). `apps:write` covering run is what keeps an app-editor + // token working without it also holding `apps:run`. + for token in ["APPS_RUN_VAPP_TOKEN", "APPS_WRITE_VAPP_TOKEN"] { + let resp = authed(client().post(format!("{base}/u/test-user/vapp")), token) + .json(&run_mode_raw_code) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "{token} must clear the scope gate for its own app and reach the policy (got {status}): {body}" + ); + assert!( + body.contains("forbidden by policy"), + "{token} must be stopped by the policy, not the scope gate, got: {body}" + ); + } + Ok(()) } diff --git a/backend/tests/app_s3_onbehalf.rs b/backend/tests/app_s3_onbehalf.rs index 7b233b505d..c11c94be1e 100644 --- a/backend/tests/app_s3_onbehalf.rs +++ b/backend/tests/app_s3_onbehalf.rs @@ -505,8 +505,8 @@ async fn seed_script(db: &Pool, path: &str, content: &str) -> anyhow:: } sqlx::query( r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, - created_by, on_behalf_of_email, language, tag, lock) - VALUES ('test-workspace', $1, $2, '', '', $3, 'test-user-2', 'test2@windmill.dev', + created_by, on_behalf_of, language, tag, lock) + VALUES ('test-workspace', $1, $2, '', '', $3, 'test-user-2', 'u/test-user-2', 'deno'::script_lang, 'deno', '') ON CONFLICT DO NOTHING"#, ) diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index 0d58615be0..0dffcc40b6 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -447,6 +447,7 @@ def main(): deployment_message: None, visible_to_runner_only: None, on_behalf_of_email: None, + on_behalf_of: None, preserve_on_behalf_of: None, ws_error_handler_muted: None, labels: None, diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs index 539eeaaa6e..366c220c62 100644 --- a/backend/tests/end_user_email.rs +++ b/backend/tests/end_user_email.rs @@ -153,6 +153,72 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res Ok(()) } +/// Create an app whose policy allows triggering the fixture flow +async fn create_app_triggering_flow(port: u16, path: &str, flow_path: &str) -> anyhow::Result<()> { + let url = format!("http://localhost:{}/api/w/test-workspace/apps/create", port); + let resp = authed(client().post(&url), SAME_WS_TOKEN) + .json(&json!({ + "path": path, + "summary": "Test app running a flow for WM_END_USER_EMAIL", + "value": { + "type": "app", + "grid": [], + "subgrids": {}, + "hiddenInlineScripts": [] + }, + "policy": { + "execution_mode": "anonymous", + "on_behalf_of": null, + "on_behalf_of_email": null, + "triggerables_v2": { + flow_path: { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!( + "create app failed: {} - {}", + resp.status(), + resp.text().await? + ); + } + Ok(()) +} + +async fn run_app_flow( + port: u16, + token: &str, + app_path: &str, + flow_path: &str, +) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", + port, app_path + ); + let resp = authed(client().post(&url), token) + .json(&json!({ + "args": {}, + "component": "run_flow", + "path": flow_path + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!( + "app flow run failed: {} - {}", + resp.status(), + resp.text().await? + ); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} + async fn run_app_inline_script( port: u16, token: &str, @@ -328,6 +394,42 @@ async fn test_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { Ok(()) } +/// Every flow step must see the same end user as the flow itself: `end_user_email` is only +/// stamped on the job the app pushes, so it has to be forwarded down to each step. The +/// fixture flow has two steps on purpose - the first one is pushed from the freshly pulled +/// flow job, every later one from a flow job re-fetched without its `job_perms`. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_app_flow_step_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let app_path = "f/test/email_flow_app"; + let flow_path = "flow/f/test/get_end_user_email_flow"; + + in_test_worker( + Connection::Sql(db.clone()), + async move { + create_app_triggering_flow(port, app_path, flow_path).await?; + + // The flow result is its last step's result. + let result = run_app_flow(port, OTHER_WS_TOKEN, app_path, flow_path).await?; + assert_eq!( + result, OTHER_WS_EMAIL, + "every flow step should see the end user email, not the app publisher's" + ); + + Ok::<(), anyhow::Error>(()) + }, + port, + ) + .await?; + + Ok(()) +} + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "end_user_email"))] async fn test_raw_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { diff --git a/backend/tests/fixtures/app_preview_auth.sql b/backend/tests/fixtures/app_preview_auth.sql index b94e08f3fd..62173d687c 100644 --- a/backend/tests/fixtures/app_preview_auth.sql +++ b/backend/tests/fixtures/app_preview_auth.sql @@ -19,6 +19,13 @@ INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) VA INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES (encode(sha256('APPS_RUN_TOKEN'::bytea), 'hex'), 'APPS_RUN_T', 'APPS_RUN_TOKEN', 'test2@windmill.dev', 'apps:run scoped token', false, '{apps:run}'); +-- Path-qualified app scopes, as the token scope picker mints them. Both must be +-- confined to `u/test-user/vapp` on the execution route (`apps:write` covers run +-- for the same app, never for another one). +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES + (encode(sha256('APPS_RUN_VAPP_TOKEN'::bytea), 'hex'), 'APPS_RUN_V', 'APPS_RUN_VAPP_TOKEN', 'test2@windmill.dev', 'apps:run path-scoped token', false, '{apps:run:u/test-user/vapp}'), + (encode(sha256('APPS_WRITE_VAPP_TOKEN'::bytea), 'hex'), 'APPS_WRIT_', 'APPS_WRITE_VAPP_TOKEN', 'test2@windmill.dev', 'apps:write path-scoped token', false, '{apps:write:u/test-user/vapp}'); + -- A private app owned by `test-user` with a persisted inline script. Used to -- assert that `test-user-2` cannot preview-execute another app's app_script id. INSERT INTO app (id, workspace_id, path, summary, policy, versions) VALUES diff --git a/backend/tests/fixtures/end_user_email.sql b/backend/tests/fixtures/end_user_email.sql index 17d459f7de..596d34e3e3 100644 --- a/backend/tests/fixtures/end_user_email.sql +++ b/backend/tests/fixtures/end_user_email.sql @@ -51,7 +51,7 @@ INSERT INTO flow (workspace_id, summary, description, path, versions, schema, va VALUES ( 'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', 'test-user', '{"g/all": true}' ); @@ -60,6 +60,6 @@ INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) VALUES ( 900002, 'test-workspace', 'f/test/get_end_user_email_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', 'test-user' ); diff --git a/backend/tests/fixtures/schedule_push.sql b/backend/tests/fixtures/schedule_push.sql index 9312d0bbd9..5dcc9e1528 100644 --- a/backend/tests/fixtures/schedule_push.sql +++ b/backend/tests/fixtures/schedule_push.sql @@ -22,13 +22,13 @@ VALUES ( 'Test script', '', 'f/system/test_script', 100001, 'deno', '', 'script' ); --- A script with on_behalf_of_email -INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, on_behalf_of_email) +-- A script with an on-behalf-of identity +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, on_behalf_of) VALUES ( 'test-workspace', 'test-user', 'export async function main() { return "obo"; }', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - 'OBO script', '', 'f/system/obo_script', 100002, 'deno', '', 'script', 'obo@windmill.dev' + 'OBO script', '', 'f/system/obo_script', 100002, 'deno', '', 'script', 'u/obo-user' ); -- A script with a tag @@ -66,13 +66,13 @@ VALUES ( 'test-user' ); --- A flow with on_behalf_of_email -INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by, on_behalf_of_email) +-- A flow with an on-behalf-of identity +INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by, on_behalf_of) VALUES ( 'test-workspace', 'OBO flow', '', 'f/system/obo_flow', '{200002}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', '{"modules": [{"id": "a", "value": {"path": "f/system/test_script", "type": "script", "input_transforms": {}}}]}', - 'test-user', 'obo@windmill.dev' + 'test-user', 'u/obo-user' ); INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) diff --git a/backend/tests/folder_default_permissioned_as.rs b/backend/tests/folder_default_permissioned_as.rs index 1e71202595..7aa4f0a003 100644 --- a/backend/tests/folder_default_permissioned_as.rs +++ b/backend/tests/folder_default_permissioned_as.rs @@ -475,17 +475,24 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu resp.text().await? ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "f/prodfolder/jobs/flow_admin_match", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("group-wm_deployers@windmill.dev"), + flow.on_behalf_of.as_deref(), + Some("g/wm_deployers"), "flow should resolve folder default to group email" ); + // The group email is synthetic and resolves back to nobody, so the group can + // only survive as the stored permissioned_as. + assert_eq!( + flow.on_behalf_of.as_deref(), + Some("g/wm_deployers"), + "flow should run as the group the folder rule names" + ); // 5b. Admin, reports/* rule (email directly as permissioned_as) let resp = authed( @@ -502,16 +509,16 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu resp.text().await? ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "f/prodfolder/reports/weekly", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), - "email rule should pass through as-is" + flow.on_behalf_of.as_deref(), + Some("u/original-user"), + "a rule naming an address stores the principal it names, which is what carries that user's groups and folders" ); // 5c. Non-matching flow path — no default @@ -524,14 +531,14 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu .await?; assert_eq!(resp.status(), 201); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "f/prodfolder/dev/flow_no_match", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email, None, + flow.on_behalf_of, None, "no folder default ⇒ no on_behalf_of_email written" ); @@ -545,14 +552,14 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu .await?; assert_eq!(resp.status(), 201); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "u/test-user/outside_flow", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email, None, + flow.on_behalf_of, None, "paths outside folders are never touched" ); @@ -575,15 +582,15 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu resp.text().await? ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", "f/prodfolder/jobs/new_script", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("group-wm_deployers@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("g/wm_deployers"), "new script at matching path gets folder default" ); diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 1a8827b486..32057b41d2 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -171,6 +171,7 @@ async fn push_job(db: &Pool, content: &str, args: &serde_json::Value) /* email */ "test@windmill.dev", /* permissioned_as */ "u/test-user".to_string(), /* token_prefix */ None, + /* audit_end_user */ None, /* scheduled_for */ None, /* schedule_path */ None, /* parent_job */ None, diff --git a/backend/tests/on_behalf_of.rs b/backend/tests/on_behalf_of.rs new file mode 100644 index 0000000000..8576bcdfe8 --- /dev/null +++ b/backend/tests/on_behalf_of.rs @@ -0,0 +1,391 @@ +//! A script's on-behalf-of identity must drive the permissions of the jobs it produces, not +//! just their address. The principal is the only stored half; the address is derived from it, +//! so a request naming one, the other, or a mismatched pair all resolve to one account. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +fn script_body(path: &str, on_behalf_of: Option<&str>) -> serde_json::Value { + let mut body = json!({ + "path": path, + "summary": "", + "description": "", + "content": "export async function main() { return 42; }", + "language": "deno", + "on_behalf_of_email": "original@windmill.dev", + "preserve_on_behalf_of": true, + "auto_parent": true, + }); + if let Some(permissioned_as) = on_behalf_of { + body["on_behalf_of"] = json!(permissioned_as); + } + body +} + +/// Deploys as `test-user` (admin) so the recorded identity is nobody's default: neither +/// the caller's nor the deployer's. Returns the hex hash the run-by-hash route parses. +async fn create_script( + base: &str, + path: &str, + on_behalf_of: Option<&str>, +) -> anyhow::Result { + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&script_body(path, on_behalf_of)) + .send() + .await?; + let status = resp.status(); + let hash = resp.text().await?; + assert_eq!(status, 201, "creating {path}: {hash}"); + Ok(hash.trim().trim_matches('"').to_string()) +} + +async fn run_by_hash(base: &str, hash: &str) -> anyhow::Result { + let resp = authed( + client().post(format!("{base}/jobs/run/h/{hash}")), + "SECRET_TOKEN", + ) + .json(&json!({})) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 201, "running {hash}: {body}"); + Ok(uuid::Uuid::parse_str(body.trim().trim_matches('"'))?) +} + +async fn stored_permissioned_as( + db: &Pool, + table: &str, + path: &str, +) -> anyhow::Result> { + // `table` is a literal from this test, never caller input. + Ok(sqlx::query_scalar(&format!( + "SELECT on_behalf_of FROM {table} \ + WHERE path = $1 AND workspace_id = 'test-workspace' AND NOT archived" + )) + .bind(path) + .fetch_one(db) + .await?) +} + +#[sqlx::test(fixtures("preserve_on_behalf_of"))] +async fn test_on_behalf_of_drives_job_identity( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + let recorded = + create_script(&base, "u/test-user/obo_recorded", Some("u/original-user")).await?; + assert_eq!( + stored_permissioned_as(&db, "script", "u/test-user/obo_recorded") + .await? + .as_deref(), + Some("u/original-user") + ); + + // Workers predating this release read only the address, and they are expected to lag the + // server, so a deploy keeps filling it in until every live worker is new. + assert_eq!( + sqlx::query_scalar!( + "SELECT on_behalf_of_email FROM script WHERE path = 'u/test-user/obo_recorded' AND workspace_id = 'test-workspace'" + ) + .fetch_one(&db) + .await? + .as_deref(), + Some("original@windmill.dev"), + ); + + // A client that predates the field names only the email; deriving the principal from + // it is what stops a routine redeploy from handing the script to whoever deploys it. + let derived = create_script(&base, "u/test-user/obo_derived", None).await?; + assert_eq!( + stored_permissioned_as(&db, "script", "u/test-user/obo_derived") + .await? + .as_deref(), + Some("u/original-user") + ); + + let recorded_job = run_by_hash(&base, &recorded).await?; + let derived_job = run_by_hash(&base, &derived).await?; + + let jobs = sqlx::query!( + "SELECT id, permissioned_as, permissioned_as_email FROM v2_job WHERE id = ANY($1)", + &[recorded_job, derived_job][..] + ) + .fetch_all(&db) + .await?; + let identity = |id: uuid::Uuid| { + let job = jobs.iter().find(|j| j.id == id).expect("job was pushed"); + ( + job.permissioned_as.clone(), + job.permissioned_as_email.clone(), + ) + }; + + assert_eq!( + identity(recorded_job), + ( + "u/original-user".to_string(), + "original@windmill.dev".to_string() + ), + "a recorded permissioned_as must be what the job runs as" + ); + assert_eq!( + identity(derived_job), + ( + "u/original-user".to_string(), + "original@windmill.dev".to_string() + ), + "a principal derived from the address drives the job the same way an explicit one does" + ); + + // A superadmin acting outside their workspaces has no `usr` row. Dropping them on an + // email-only redeploy would keep their superadmin email next to the deployer's + // permissions — the hybrid identity this whole change exists to remove. + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": "u/test-user/obo_superadmin", + "summary": "", + "description": "", + "content": "export async function main() { return 42; }", + "language": "deno", + "on_behalf_of_email": "superadmin-external@windmill.dev", + "preserve_on_behalf_of": true, + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "creating: {}", resp.text().await?); + assert_eq!( + stored_permissioned_as(&db, "script", "u/test-user/obo_superadmin") + .await? + .as_deref(), + Some("u/superadmin-external") + ); + + // The synthetic group namespace is not reserved, so a real account holding such an + // address must win over the like-named group — otherwise an email-only deploy would + // hand the runnable that group's folder access. + sqlx::query!( + "INSERT INTO group_ (workspace_id, name, summary, extra_perms) \ + VALUES ('test-workspace', 'ops', '', '{}') ON CONFLICT DO NOTHING" + ) + .execute(&db) + .await?; + sqlx::query!( + "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE workspace_id = 'test-workspace' \ + AND username = 'test-user-2'" + ) + .execute(&db) + .await?; + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": "u/test-user/obo_group_collision", + "summary": "", + "description": "", + "content": "export async function main() { return 42; }", + "language": "deno", + "on_behalf_of_email": "group-ops@windmill.dev", + "preserve_on_behalf_of": true, + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "creating: {}", resp.text().await?); + assert_eq!( + stored_permissioned_as(&db, "script", "u/test-user/obo_group_collision") + .await? + .as_deref(), + Some("u/test-user-2"), + "a real account must win over the like-named group" + ); + + // An address is the principal only for an account whose username is that address; for + // anybody else it is canonicalized, since the bare form carries neither their groups nor + // their folders. + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&script_body( + "u/test-user/obo_bare_address", + Some("original@windmill.dev"), + )) + .send() + .await?; + assert_eq!(resp.status(), 201, "creating: {}", resp.text().await?); + assert_eq!( + stored_permissioned_as(&db, "script", "u/test-user/obo_bare_address") + .await? + .as_deref(), + Some("u/original-user") + ); + + // A job row carries a narrower identity column than the runnable it comes from, so an + // address too long to be enqueued is refused at deploy rather than at the first run — + // whether it is preserved from someone else or is the deployer's own. + const LONG_ADDRESS: &str = "a-very-long-superadmin-address-for-this-test@windmill.dev"; + sqlx::query!( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ($1, '', 'password', true, true, '')", + LONG_ADDRESS + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('LONG_TOKEN'::bytea), 'hex'), 'LONG_TOKEN', 'LONG_TOKEN', $1, 'long', true)", + LONG_ADDRESS + ) + .execute(&db) + .await?; + + let too_long = |path: &str| { + json!({ + "path": path, + "summary": "", + "description": "", + "content": "export async function main() { return 42; }", + "language": "deno", + "on_behalf_of_email": LONG_ADDRESS, + "preserve_on_behalf_of": true, + }) + }; + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&too_long("u/test-user/obo_too_long")) + .send() + .await?; + assert_eq!(resp.status(), 400, "an unenqueueable identity must be rejected"); + + // Picking "me" does not preserve anyone, so it takes the branch that stores the caller's + // own principal — which for an account acting without a `usr` row is their address. + let mut own = too_long("u/test-user/obo_too_long_self"); + own["preserve_on_behalf_of"] = json!(false); + let resp = authed( + client().post(format!("{base}/scripts/create")), + "LONG_TOKEN", + ) + .json(&own) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("characters a job can carry"), + "the caller's own identity has to be refused by the same check, not by a column \ + overflow further down: {body}" + ); + + // A pair naming two different principals would run as a composite of both. + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&script_body( + "u/test-user/obo_mismatch", + Some("u/test-user-2"), + )) + .send() + .await?; + assert_eq!(resp.status(), 400, "a mismatched pair must be rejected"); + + // The identity a no-op push is compared against is the stored principal, so a push that + // names it by address alone still has to read as unchanged — otherwise every idempotent + // CLI push of a configured script would cut a version and a phantom git-sync commit. + async fn push_noop_guarded(base: &str, body: serde_json::Value) -> anyhow::Result { + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let hash = resp.text().await?; + assert_eq!(status, 201, "creating: {hash}"); + Ok(hash.trim().trim_matches('"').to_string()) + } + // The no-op check compares every field, so the body has to carry the values a deploy + // fills in by itself, or it would be rejected before reaching the identity comparison. + let noop_body = |permissioned_as| { + let mut body = script_body("u/test-user/obo_noop", permissioned_as); + body["ws_error_handler_muted"] = json!(false); + body["assets"] = json!([]); + body + }; + let first = push_noop_guarded(&base, noop_body(Some("u/original-user"))).await?; + let again = push_noop_guarded(&base, noop_body(None)).await?; + assert_eq!( + first, again, + "an identical push naming the same identity by address must not cut a new version" + ); + + // Flows resolve the same way, but through their own UPDATE — which must not drop the + // principal when the body names only the email. + let flow = json!({ + "path": "u/test-user/obo_flow", + "summary": "", + "value": { "modules": [] }, + "on_behalf_of_email": "original@windmill.dev", + "on_behalf_of": "u/original-user", + "preserve_on_behalf_of": true, + }); + let resp = authed( + client().post(format!("{base}/flows/create")), + "SECRET_TOKEN", + ) + .json(&flow) + .send() + .await?; + assert_eq!(resp.status(), 201, "creating flow: {}", resp.text().await?); + + let mut update = flow.clone(); + update["summary"] = json!("edited"); + update + .as_object_mut() + .unwrap() + .remove("on_behalf_of"); + let resp = authed( + client().post(format!("{base}/flows/update/u/test-user/obo_flow")), + "SECRET_TOKEN", + ) + .json(&update) + .send() + .await?; + assert_eq!(resp.status(), 200, "updating flow: {}", resp.text().await?); + assert_eq!( + stored_permissioned_as(&db, "flow", "u/test-user/obo_flow") + .await? + .as_deref(), + Some("u/original-user"), + "an update that names only the email must not drop the flow's principal" + ); + + Ok(()) +} diff --git a/backend/tests/postgres_trigger_scope.rs b/backend/tests/postgres_trigger_scope.rs index 1172e03ab6..cf98a1e6de 100644 --- a/backend/tests/postgres_trigger_scope.rs +++ b/backend/tests/postgres_trigger_scope.rs @@ -20,6 +20,8 @@ fn scoped_authed(scopes: Vec<&str>) -> ApiAuthed { folders: vec![], scopes: Some(scopes.into_iter().map(str::to_string).collect()), username_override: None, + username_override_is_token_label: false, + is_session_token: false, token_prefix: None, read_only: false, } diff --git a/backend/tests/preserve_on_behalf_of.rs b/backend/tests/preserve_on_behalf_of.rs index d808348937..e696de44ea 100644 --- a/backend/tests/preserve_on_behalf_of.rs +++ b/backend/tests/preserve_on_behalf_of.rs @@ -214,15 +214,15 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2", "u/test-user/script_admin_preserve", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("u/original-user"), "Admin should preserve on_behalf_of_email" ); @@ -249,15 +249,15 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2", "u/deployer-user/script_deployer_preserve", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("u/original-user"), "Deployer should preserve on_behalf_of_email" ); @@ -284,15 +284,15 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2", "u/test-user-2/script_no_preserve", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("test2@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("u/test-user-2"), "Non-admin should have their own email as on_behalf_of_email" ); @@ -319,15 +319,15 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "u/test-user/flow_admin_preserve", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + flow.on_behalf_of.as_deref(), + Some("u/original-user"), "Admin should preserve flow on_behalf_of_email" ); @@ -354,15 +354,15 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "u/deployer-user/flow_deployer_preserve", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + flow.on_behalf_of.as_deref(), + Some("u/original-user"), "Deployer should preserve flow on_behalf_of_email" ); @@ -389,15 +389,15 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "u/test-user-2/flow_no_preserve", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("test2@windmill.dev"), + flow.on_behalf_of.as_deref(), + Some("u/test-user-2"), "Non-admin should have their own email as flow on_behalf_of_email" ); @@ -745,15 +745,15 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2", "u/test-user/script_no_flag", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("test@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("u/test-user"), "Without preserve flag, admin's email should be used" ); @@ -841,15 +841,15 @@ async fn test_script_update_preserves_on_behalf_of(db: Pool) -> anyhow ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", "u/original-user/script_to_update", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("u/original-user"), "Admin update should preserve script on_behalf_of_email" ); @@ -896,15 +896,15 @@ async fn test_script_update_preserves_on_behalf_of(db: Pool) -> anyhow ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", "u/deployer-user/script_deploy_update", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("u/original-user"), "Deployer update should preserve script on_behalf_of_email" ); @@ -951,15 +951,15 @@ async fn test_script_update_preserves_on_behalf_of(db: Pool) -> anyhow ); let script = sqlx::query!( - "SELECT on_behalf_of_email FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "SELECT on_behalf_of FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", "u/test-user-2/script_nonadmin_update", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - script.on_behalf_of_email.as_deref(), - Some("test2@windmill.dev"), + script.on_behalf_of.as_deref(), + Some("u/test-user-2"), "Non-admin update should overwrite script on_behalf_of_email with their own" ); @@ -1025,15 +1025,15 @@ async fn test_flow_update_preserves_on_behalf_of(db: Pool) -> anyhow:: ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "u/original-user/flow_to_update", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + flow.on_behalf_of.as_deref(), + Some("u/original-user"), "Admin update should preserve flow on_behalf_of_email" ); @@ -1091,15 +1091,15 @@ async fn test_flow_update_preserves_on_behalf_of(db: Pool) -> anyhow:: ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "u/deployer-user/flow_deploy_update", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("original@windmill.dev"), + flow.on_behalf_of.as_deref(), + Some("u/original-user"), "Deployer update should preserve flow on_behalf_of_email" ); @@ -1157,15 +1157,15 @@ async fn test_flow_update_preserves_on_behalf_of(db: Pool) -> anyhow:: ); let flow = sqlx::query!( - "SELECT on_behalf_of_email FROM flow WHERE path = $1 AND workspace_id = $2", + "SELECT on_behalf_of FROM flow WHERE path = $1 AND workspace_id = $2", "u/test-user-2/flow_nonadmin_update", "test-workspace" ) .fetch_one(&db) .await?; assert_eq!( - flow.on_behalf_of_email.as_deref(), - Some("test2@windmill.dev"), + flow.on_behalf_of.as_deref(), + Some("u/test-user-2"), "Non-admin update should overwrite flow on_behalf_of_email with their own" ); diff --git a/backend/tests/preview_native_tag.rs b/backend/tests/preview_native_tag.rs index 29aefa588a..eba686396a 100644 --- a/backend/tests/preview_native_tag.rs +++ b/backend/tests/preview_native_tag.rs @@ -56,6 +56,7 @@ async fn push_preview_and_get_row( /* email */ "test@windmill.dev", /* permissioned_as */ "u/test-user".to_string(), /* token_prefix */ None, + /* audit_end_user */ None, /* scheduled_for */ None, /* schedule_path */ None, /* parent_job */ None, diff --git a/backend/tests/trigger_listener_queries.rs b/backend/tests/trigger_listener_queries.rs index c3f6357d4c..52088557ac 100644 --- a/backend/tests/trigger_listener_queries.rs +++ b/backend/tests/trigger_listener_queries.rs @@ -173,6 +173,8 @@ fn make_authed() -> windmill_api_auth::ApiAuthed { folders: vec![], scopes: None, username_override: None, + username_override_is_token_label: false, + is_session_token: false, token_prefix: None, read_only: false, } diff --git a/backend/windmill-api-assets/Cargo.toml b/backend/windmill-api-assets/Cargo.toml index c75715a3ac..62bc3d9c8d 100644 --- a/backend/windmill-api-assets/Cargo.toml +++ b/backend/windmill-api-assets/Cargo.toml @@ -21,3 +21,7 @@ serde.workspace = true serde_json.workspace = true sqlx.workspace = true tracing.workspace = true +uuid.workspace = true + +[dev-dependencies] +tokio = { workspace = true } diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 3a2bf83097..09720b2203 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -605,9 +605,16 @@ async fn list_favorites( // frontend aggregates into nodes and edges. #[derive(Deserialize)] -struct GraphQuery { +pub struct GraphQuery { pub asset_kinds: Option, pub folder: Option, + /// Render a dbt project as a given deployed version rather than as it is + /// now. A run page passes the version its job ran, so an old run shows the + /// models, SQL and `ref()` lineage of that deploy instead of today's. + /// Absent — the usual case — means the newest live version per path. + /// Hex, like every other script-hash parameter — `ScriptHash` deserializes + /// it, so the run page can pass `job.script_hash` verbatim. + pub dbt_script_hash: Option, } #[derive(Serialize, Debug)] @@ -629,6 +636,70 @@ struct GraphAssetNode { // `AssetGraphAssetNode.derived_from`. #[serde(skip_serializing_if = "Option::is_none")] derived_from: Option, + // Set on a `dbt://` asset produced (or, for a source, consumed) by a dbt + // script: which dbt node it is and what dbt says about it. A dbt project is + // one runnable node with many model assets, so per-model metadata belongs + // here rather than on the script (docs/dbt-runtime.md, decision 15). + // Lockstep with TS `AssetGraphAssetNode.dbt`. + #[serde(skip_serializing_if = "Option::is_none")] + dbt: Option, +} + +#[derive(Serialize, Debug, Clone)] +struct DbtAssetProvenance { + unique_id: String, + // `model` | `snapshot` | `seed` | `source` — a source is read, not written, + // and the canvas distinguishes them. + resource_type: String, + // dbt's own word (`table`, `view`, `incremental`, `snapshot`), kept because + // `view` and `ephemeral` have no Windmill write-strategy analogue. + #[serde(skip_serializing_if = "Option::is_none")] + materialized: Option, + #[serde(skip_serializing_if = "Option::is_none")] + materialize_strategy: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + tags: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + data_tests: Vec, + /// Declared column metadata (name -> description). NOT column lineage — + /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14). + #[serde(skip_serializing_if = "Option::is_none")] + columns: Option, + /// A source's declared freshness policy, for the staleness chip. + #[serde(skip_serializing_if = "Option::is_none")] + freshness: Option, + /// The model's SQL as written. Read-only in Windmill — the file lives in + /// the producing script's bundle, and this is the copy captured when the + /// graph being read was parsed: at deploy, or by a refresh from the + /// editor's buffer. + #[serde(skip_serializing_if = "Option::is_none")] + raw_code: Option, + /// Its path inside the repo, e.g. `models/staging/stg_orders.sql`. + #[serde(skip_serializing_if = "Option::is_none")] + original_file_path: Option, +} + +#[derive(Serialize, Debug, Clone, PartialEq)] +struct DbtDataTest { + // `unique` | `not_null` | `accepted_values` | `relationships` — the four + // generic tests, one-for-one with the `// data_test` kinds — or a package + // test's namespaced name (`dbt_utils.accepted_range`). + kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + column: Option, + #[serde(skip_serializing_if = "Option::is_none")] + args: Option, + // Lowercased. dbt's own severity decides whether a failure fails the run, + // so the canvas shows it rather than assuming every test is blocking. + #[serde(skip_serializing_if = "Option::is_none")] + severity: Option, +} + +#[derive(Serialize, Debug, Clone)] +struct DbtRunnableProvenance { + model_count: usize, } #[derive(Serialize, Debug)] @@ -691,6 +762,11 @@ struct GraphRunnableNode { // signature list. Lockstep with TS `AssetGraphRunnableNode.macros`. #[serde(skip_serializing_if = "Vec::is_empty", default)] macros: Vec, + // Set on a `ScriptLang::Dbt` script: it owns a whole dbt project, so the + // node says how many models it materializes rather than pretending to be a + // single-output script. Lockstep with TS `AssetGraphRunnableNode.dbt`. + #[serde(skip_serializing_if = "Option::is_none", default)] + dbt: Option, } // One macro of a `// macros` library, as surfaced on its graph node. @@ -831,7 +907,7 @@ struct TestEdge { } #[derive(Serialize, Debug)] -struct AssetGraphResponse { +pub struct AssetGraphResponse { assets: Vec, runnables: Vec, edges: Vec, @@ -840,6 +916,34 @@ struct AssetGraphResponse { macro_edges: Vec, #[serde(skip_serializing_if = "Vec::is_empty", default)] test_edges: Vec, + /// `ref()` lineage BETWEEN two dbt models. Without it every model hangs off + /// the one dbt runnable, which reads as a flat source-to-model fan-out + /// instead of the project's actual shape. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + dbt_edges: Vec, + /// The job whose snapshot the dbt half was resolved from, when one was + /// asked for and found. A run page polls the graph while its job runs + /// because a dynamic descriptor's snapshot is written mid-run, and this is + /// what tells it to stop: without it the page cannot distinguish "the + /// snapshot has not been written yet" from "this run has none", and either + /// polls forever or gives up before the ingest. + #[serde(skip_serializing_if = "Option::is_none")] + dbt_snapshot_job: Option, + /// When the dbt half was parsed, for a graph pinned to a job. What the + /// editor labels its provenance with: a buffer refresh and the deployed + /// version's graph are drawn identically, so without saying which one is on + /// screen the ambiguity the explicit refresh removes just moves into the + /// editor. + #[serde(skip_serializing_if = "Option::is_none")] + dbt_graph_ingested_at: Option>, +} + +/// One `ref()`/`source()` edge, in the terms the canvas draws: the two +/// relations, not dbt's node ids. +#[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct DbtLineageEdge { + from_asset_path: String, + to_asset_path: String, } async fn asset_graph( @@ -849,15 +953,81 @@ async fn asset_graph( Extension(db): Extension, Query(q): Query, ) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; + // `None`: pinning the graph to one run is job-scoped and this endpoint is + // authorized as `assets:read`. See `asset_graph_for`. + asset_graph_for(&authed, &w_id, user_db, db, q, None).await +} - let kind_filter: Option> = q.asset_kinds.as_ref().map(|s| { - s.split(',') - .filter_map(|k| { - serde_json::from_value::(Value::String(k.trim().into())).ok() - }) - .collect() - }); +/// A run the caller is already authorized to read, and the deployed version it +/// ran. Path and hash come from the job row rather than the query, so a graph +/// cannot be pointed at one project's version while claiming another's run. +pub struct PinnedRun { + pub job_id: uuid::Uuid, + pub script_path: String, + /// `None` for a job that names no deployed version: a `parse` of the + /// EDITOR's buffer, whose graph belongs to that job alone. Such a job pins + /// only because it stored one — every other preview answers with the + /// workspace graph, as before. + pub script_hash: Option, +} + +/// The asset graph, optionally as one run saw it. +/// +/// AUTHORIZES NOTHING BY ITSELF. Every caller owes it two checks, because the +/// answer is workspace asset data reached through a route whose own URL segment +/// decides the scope domain: `assets:read` always, and `require_job_read_access` +/// for the job when passing `Some(pinned)` — which is then taken as already +/// done, since the pinned path and hash come from that job's row. +pub async fn asset_graph_for( + authed: &ApiAuthed, + w_id: &str, + user_db: UserDB, + db: windmill_common::DB, + q: GraphQuery, + pinned: Option, +) -> JsonResult { + let dbt_job_id = pinned.as_ref().map(|p| p.job_id); + // The version is the job's own, not the caller's `dbt_script_hash` — and + // that holds when the job names NONE, so the parameter cannot supply a + // version for a graph that has no business claiming one. + let dbt_script_hash = match pinned.as_ref() { + Some(p) => p.script_hash, + None => q.dbt_script_hash.map(|h| h.0), + }; + // Set only for a pinned run, where it lets `live` resolve without reading + // `script`: a share-link viewer is entitled to the run but usually has no + // grant on the script, and RLS there would empty the graph. `raw_code` has + // its own `script` check, so the body stays hidden either way. + let pinned_path = pinned.as_ref().map(|p| p.script_path.as_str()); + let w_id = w_id.to_string(); + let mut tx = user_db.begin(authed).await?; + // Built once: a scoped token's `scripts:read` paths decide whether a dbt + // node's SQL body may be returned, independently of the `assets:read` scope + // that authorizes this endpoint. + let dbt_source_scope = build_scope_path_predicate(authed, "scripts", "read"); + + // REFUSED, not dropped: a caller that asks for a kind this server does not + // know has asked for something, and answering with the other kinds returns a + // graph missing exactly what they came for — silently. That is how a rename + // of one kind emptied three callers' graphs with nothing logged. + let kind_filter: Option> = q + .asset_kinds + .as_ref() + .map(|s| { + s.split(',') + .map(|k| { + serde_json::from_value::(Value::String(k.trim().into())).map_err( + |_| { + windmill_common::error::Error::BadRequest(format!( + "`{}` is not an asset kind", + k.trim() + )) + }, + ) + }) + .collect::, _>>() + }) + .transpose()?; let kind_filter_ref = kind_filter.as_deref(); let folder_filter = q.folder.as_deref().map(|f| format!("f/{}/%", f)); @@ -1066,6 +1236,267 @@ async fn asset_graph( .fetch_all(&mut *tx) .await?; + // dbt provenance. A dbt script is one runnable node whose models are many + // `dbt://` asset nodes (decision 15), so the per-model metadata has to + // hang off the assets, not off the script. It describes the RELATION, not a + // producer: several dbt scripts (different selections of one project) can + // materialize the same model, and the producer edges already name them. + // + // Scoped to the relations this graph actually renders, so a workspace with + // many dbt projects does not pay for all of them on every request. Not + // scoped by the script's folder: an out-of-folder dbt script still has to + // explain a model an in-scope consumer reads, same as macro definitions. + // Tests come along via `attached_node` — they carry no `asset_path` of + // their own. + let dbt_rows = sqlx::query!( + r#"WITH live AS ( + -- The graph is stored per deployed VERSION, so this endpoint — which + -- describes the project as it is now — takes the newest live one per + -- path. Resolved once here rather than per row: a correlated lookup + -- on every node is what makes these queries fall over. + SELECT * FROM ( + SELECT DISTINCT ON (s.path) s.path, s.hash + FROM script s + WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt' + AND ($3::bigint IS NULL OR s.hash = $3) + -- A pinned version may be archived by now; that is precisely + -- the case a historical run needs, so the liveness filter + -- applies only when picking the current one. + AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false)) + ORDER BY s.path, s.created_at DESC + ) cur + UNION ALL + -- A pinned run names its own version, so `script` is not consulted: + -- under RLS it would answer for the CALLER's grants on the project, + -- emptying the graph for a share-link viewer who is entitled to the + -- run but not the script. A NULL hash here is a job that names no + -- version at all — an editor buffer parse — and matches only the + -- version-less rows that parse stored. + SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL + ), + -- The run's own snapshot when it left one, the version's graph + -- otherwise. A static descriptor never snapshots, so all of its runs + -- fall through to the same rows. Existence comes from the marker, not + -- from a node row: a dynamic run that disabled every model has a + -- snapshot whose graph is legitimately empty. + chosen AS ( + -- No visibility check on the job here: reaching this with a job at + -- all means the caller passed `require_job_read_access` for it, and + -- re-deciding it under plain RLS can only DISAGREE with that answer + -- — silently, by falling back to the deployed graph rather than + -- erroring. A share-link viewer is entitled to the run and would be + -- shown a different run's model set. See `asset_graph_for`. + SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $4) + THEN $4::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id + ), + scoped AS ( + SELECT n.script_path, n.unique_id FROM dbt_node n + -- `=` still, with the NULL-to-NULL case spelled out and gated on + -- the pin: a version-less row's hash is NULL on both sides, which + -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the + -- equality its index bound on the UNPINNED workspace graph — the + -- hot path. Unpinned, `$5` is NULL and the second arm folds away. + JOIN live l ON l.path = n.script_path + AND (n.script_hash = l.hash + OR ($5::text IS NOT NULL AND l.hash IS NULL + AND n.script_hash IS NULL)) + JOIN chosen ch ON ch.job_id = n.job_id + WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL + -- Unpinned, the scope is the relations in view: `asset` says + -- which of them this folder touches. Pinned, that table is the + -- WRONG scope — it holds one row set per path, describing the + -- current deploy, so a model this version had and the current one + -- dropped would be filtered out of its own run's graph. The + -- pinned graph's nodes are the scope. Keyed on the pin rather + -- than on the hash: an editor parse pins without naming one, and + -- its models are precisely the ones `asset` does not know yet. + AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL + OR n.asset_path IN ( + SELECT path FROM asset + WHERE workspace_id = $1 AND kind = 'dbt' + AND ($2::text IS NULL OR usage_path LIKE $2))) + ) + SELECT n.script_path AS "script_path!", n.unique_id AS "unique_id!", + n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path, + n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description, + n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node, + n.columns, n.freshness, + n.raw_code, n.original_file_path, + -- Whether the caller may read the project this row describes. + -- The query deliberately reaches outside the requested folder + -- so an in-scope consumer can explain the relation it reads, + -- and `dbt_node` carries no RLS of its own; the relation's + -- SHAPE is fine to answer that way, everything the project's + -- author WROTE is not. Applied in Rust, over one predicate, so + -- the fields it covers are named in one place. This runs in the + -- authed transaction, so `script`'s RLS answers it. Matched on + -- the HASH as well: `extra_perms` is per row, so a path + -- recreated with narrower ones leaves the archived version + -- readable, and a path-only probe would answer for THAT grant + -- while returning this version's source. + -- + -- A version-less row has no `script` row to ask, and needs + -- none: it exists only because this caller's own parse job + -- created it from a buffer they wrote, and the unpinned `live` + -- branch — fed from `script` — can never join to one. + (n.script_hash IS NULL OR EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path + AND sc.hash = n.script_hash + )) AS "script_visible!" + FROM dbt_node n + JOIN live l ON l.path = n.script_path + AND (n.script_hash = l.hash + OR ($5::text IS NOT NULL AND l.hash IS NULL + AND n.script_hash IS NULL)) + -- Every join onto `dbt_node` needs this, not just the scoping CTE: + -- `job_id` is part of the key, so without it each model comes back + -- once per retained snapshot plus once for the version's graph. + JOIN chosen ch ON ch.job_id = n.job_id + WHERE n.workspace_id = $1 + -- Joined on BOTH columns: a dbt `unique_id` is project-local, so + -- two projects with the same model name would otherwise pull each + -- other's rows. + AND (EXISTS (SELECT 1 FROM scoped s + WHERE s.script_path = n.script_path + AND s.unique_id = n.unique_id) + OR EXISTS (SELECT 1 FROM scoped s + WHERE s.script_path = n.script_path + AND s.unique_id = n.attached_node)) + ORDER BY n.script_path, n.unique_id"#, + &w_id, + folder_filter.as_deref(), + dbt_script_hash, + dbt_job_id, + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + // `ref()` lineage between two models, resolved to the relations they + // produce. Joined to `dbt_node` on both key columns because a dbt + // `unique_id` is only unique within its project. + let dbt_edge_rows = sqlx::query!( + r#"WITH live AS ( + SELECT * FROM ( + SELECT DISTINCT ON (s.path) s.path, s.hash + FROM script s + WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt' + AND ($3::bigint IS NULL OR s.hash = $3) + AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false)) + ORDER BY s.path, s.created_at DESC + ) cur + UNION ALL + SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL + ), + -- The run's own snapshot when it left one, the version's graph + -- otherwise. A static descriptor never snapshots, so all of its runs + -- fall through to the same rows. Existence comes from the marker, not + -- from a node row: a dynamic run that disabled every model has a + -- snapshot whose graph is legitimately empty. + chosen AS ( + -- No visibility check on the job here: reaching this with a job at + -- all means the caller passed `require_job_read_access` for it, and + -- re-deciding it under plain RLS can only DISAGREE with that answer + -- — silently, by falling back to the deployed graph rather than + -- erroring. A share-link viewer is entitled to the run and would be + -- shown a different run's model set. See `asset_graph_for`. + SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $4) + THEN $4::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id + ) + SELECT p.asset_path AS "from_path!", c.asset_path AS "to_path!" + FROM dbt_edge e + JOIN live l ON l.path = e.script_path + AND (e.script_hash = l.hash + OR ($5::text IS NOT NULL AND l.hash IS NULL + AND e.script_hash IS NULL)) + JOIN chosen ch ON ch.job_id = e.job_id + -- Gated the same way as the `live` joins above, and for the same + -- reason twice over: unpinned this folds to a plain equality the + -- versioned key can bound, and pinned it is the only way an editor + -- graph's NULL-to-NULL hashes meet at all. + JOIN dbt_node p ON p.workspace_id = e.workspace_id + AND p.script_path = e.script_path + AND (p.script_hash = e.script_hash + OR ($5::text IS NOT NULL AND e.script_hash IS NULL + AND p.script_hash IS NULL)) + AND p.job_id = ch.job_id + AND p.unique_id = e.parent_unique_id + JOIN dbt_node c ON c.workspace_id = e.workspace_id + AND c.script_path = e.script_path + AND (c.script_hash = e.script_hash + OR ($5::text IS NOT NULL AND e.script_hash IS NULL + AND c.script_hash IS NULL)) + AND c.job_id = ch.job_id + AND c.unique_id = e.child_unique_id + WHERE e.workspace_id = $1 + AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL + -- Tests attach to their model as a badge, not as a lineage edge. + AND c.resource_type <> 'test' + -- Scoped by the RELATIONS, like the node provenance above, not by + -- the producing script's folder: two tables consumed in this + -- folder but produced by a dbt project outside it would otherwise + -- both render with their `ref()` edge missing. + -- Same as the node scope: pinned, `asset` describes the CURRENT + -- deploy, so gating on it drops the edges of models this version + -- had and a later one removed. The pinned graph's own edges are + -- the answer. + AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL OR EXISTS ( + SELECT 1 FROM asset a + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.path = c.asset_path + AND ($2::text IS NULL OR a.usage_path LIKE $2)))"#, + &w_id, + folder_filter.as_deref(), + dbt_script_hash, + dbt_job_id, + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + // The marker `chosen` resolved to, answered once for the caller: its + // EXISTENCE is what lets a run page stop polling, and its timestamp is what + // an editor labels the graph's provenance with — "parsed from the editor at + // 14:32" against "as of last deploy". No `v2_job` recheck, for the reason + // `chosen` gives — disagreeing with the route's decision here would leave a + // share-link viewer with the right graph and a null marker, polling it 40 + // times over. + let dbt_marker = match (dbt_job_id, pinned_path) { + (Some(job), Some(path)) => { + sqlx::query!( + "SELECT g.job_id, g.ingested_at FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.script_path = $2 + AND g.script_hash IS NOT DISTINCT FROM $3 + AND g.job_id = CASE WHEN EXISTS ( + SELECT 1 FROM dbt_graph_snapshot s + WHERE s.workspace_id = $1 AND s.job_id = $4) + THEN $4::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END + LIMIT 1", + &w_id, + path, + dbt_script_hash, + job, + ) + .fetch_optional(&mut *tx) + .await? + } + _ => None, + }; + // Only a graph of the RUN's own, never the version's fallback: the run page + // compares this against its job id to decide whether to keep polling. + let dbt_snapshot_job = dbt_marker + .as_ref() + .map(|m| m.job_id) + .filter(|j| !j.is_nil()); + let dbt_graph_ingested_at = dbt_marker.as_ref().map(|m| m.ingested_at); tx.commit().await?; // Parse each pipeline member's body once into its badge annotations, keyed @@ -1110,6 +1541,138 @@ async fn asset_graph( (r.path.clone(), lineage) }) .collect(); + // Model → its provenance, and script → how many models it owns. Tests are + // folded onto the model they are attached to, which is what makes dbt's + // four generic tests render through the existing data-test node. + let mut dbt_by_asset_path: std::collections::HashMap = + Default::default(); + let mut dbt_model_count: std::collections::HashMap = Default::default(); + // Which relations each dbt script actually BUILDS. The sidecar also holds a + // row for a parent kept only to anchor a cross-selection edge, which the + // script reads rather than materializes — counting those would make a script + // selecting one mart claim the staging models upstream of it, and the badge + // says "materializes N models". + let dbt_writes: std::collections::HashSet<(&str, &str)> = rows + .iter() + .filter(|r| { + r.asset_kind == AssetKind::Dbt + && matches!(r.access_type.as_deref(), Some("w") | Some("rw")) + }) + .map(|r| (r.usage_path.as_str(), r.asset_path.as_str())) + .collect(); + // Keyed by (script_path, unique_id) — the sidecar's own primary key — since + // a dbt `unique_id` is only unique within its project. + let dbt_asset_path_by_unique_id: std::collections::HashMap<(&str, &str), &str> = dbt_rows + .iter() + .filter_map(|r| { + Some(( + (r.script_path.as_str(), r.unique_id.as_str()), + r.asset_path.as_deref()?, + )) + }) + .collect(); + for r in &dbt_rows { + let Some(asset_path) = r.asset_path.as_deref() else { + continue; + }; + if r.resource_type != "source" && dbt_writes.contains(&(r.script_path.as_str(), asset_path)) + { + *dbt_model_count.entry(r.script_path.clone()).or_default() += 1; + } + // What the project's author WROTE — the model's SQL, its path in the + // repo, the prose and labels around it — as opposed to the shape of the + // relation it produces. RLS says whether the caller may see the script, + // but not whether a scoped token may: this endpoint is authorized as + // `assets:read`, so a token deliberately narrowed to it would otherwise + // read a project outside its `scripts:read` paths. Both answers gate the + // same set of fields, because a share-link viewer entitled to the RUN is + // not thereby entitled to the documentation of a project they cannot + // open. + let source_allowed = r.script_visible && dbt_source_scope(&r.script_path); + let candidate = DbtAssetProvenance { + raw_code: r.raw_code.clone().filter(|_| source_allowed), + original_file_path: r.original_file_path.clone().filter(|_| source_allowed), + unique_id: r.unique_id.clone(), + resource_type: r.resource_type.clone(), + materialized: r.materialized.clone(), + materialize_strategy: r.materialize_strategy.clone(), + tags: if source_allowed { + r.tags.clone() + } else { + vec![] + }, + description: r.description.clone().filter(|_| source_allowed), + data_tests: vec![], + columns: r.columns.clone().filter(|_| source_allowed), + freshness: r.freshness.clone().filter(|_| source_allowed), + }; + // One relation can carry rows from several projects — typically a model + // in one and a source declaring it in another. The producer describes + // the relation and the source only names it, so which one wins must not + // depend on script-path ordering. The source's freshness policy is + // still worth keeping, so it fills in rather than overwrites. + let existing = dbt_by_asset_path + .entry(asset_path.to_string()) + .or_insert_with(|| candidate.clone()); + if existing.unique_id == candidate.unique_id { + continue; + } + let wins = match ( + existing.resource_type.as_str(), + candidate.resource_type.as_str(), + ) { + ("source", t) if t != "source" => true, + (t, "source") if t != "source" => false, + // Two rows of the same nature: pick by id so the graph does not + // change shape between requests. + _ => candidate.unique_id < existing.unique_id, + }; + let freshness = existing + .freshness + .clone() + .or_else(|| candidate.freshness.clone()); + if wins { + *existing = candidate; + } + existing.freshness = freshness; + } + // Tests fold onto the model they assert, which is what makes dbt's four + // generic tests render through the existing data-test node. A separate pass + // because rows arrive in `unique_id` order, so a test can precede its model. + for r in &dbt_rows { + if r.resource_type != "test" { + continue; + } + let Some(target) = r + .attached_node + .as_deref() + .and_then(|n| dbt_asset_path_by_unique_id.get(&(r.script_path.as_str(), n))) + else { + continue; + }; + let Some(entry) = dbt_by_asset_path.get_mut(*target) else { + continue; + }; + let test = DbtDataTest { + kind: r.test_kind.clone().unwrap_or_else(|| r.name.clone()), + column: r.test_column.clone(), + // The test's kind and column are the badge's shape, already spelled + // out by its `unique_id`. Its arguments are authored data — an + // `accepted_values` list is a column's domain — so they follow the + // same gate as the model's own source. + args: r + .test_args + .clone() + .filter(|_| r.script_visible && dbt_source_scope(&r.script_path)), + // dbt-core 1.x echoes the author's casing, 2.x uppercases; fold so + // the badge reads the same whichever engine deployed the script. + severity: r.severity.as_deref().map(|s| s.to_ascii_lowercase()), + }; + if !entry.data_tests.contains(&test) { + entry.data_tests.push(test); + } + } + let last_success_by_path: std::collections::HashMap> = last_success_rows .into_iter() @@ -1132,6 +1695,18 @@ async fn asset_graph( let mut edges = Vec::with_capacity(rows.len()); let mut asset_set: std::collections::HashSet<(AssetKind, String)> = Default::default(); + // Pinned, the relations come from that graph's own nodes. The `asset` rows + // above are path-keyed — one set per script, always the current deploy — so + // a model this version had and a later one dropped would be missing from its + // own run's graph, and an editor buffer's new models would be missing + // outright, `asset` having never heard of them. + if dbt_script_hash.is_some() || pinned_path.is_some() { + for r in &dbt_rows { + if let Some(p) = r.asset_path.as_deref() { + asset_set.insert((AssetKind::Dbt, p.to_string())); + } + } + } let mut runnable_set: std::collections::HashSet<(AssetUsageKind, String)> = Default::default(); // Every pipeline member in scope goes into the graph, even when the parser @@ -1491,6 +2066,9 @@ async fn asset_graph( }) .flatten(), derived_from: scd2_current_base.get(&(kind, path.clone())).cloned(), + dbt: (kind == AssetKind::Dbt) + .then(|| dbt_by_asset_path.get(&path).cloned()) + .flatten(), kind, path, }) @@ -1562,6 +2140,10 @@ async fn asset_graph( .flatten() .cloned() .unwrap_or_default(), + dbt: (usage_kind == AssetUsageKind::Script) + .then(|| dbt_model_count.get(&path)) + .flatten() + .map(|n| DbtRunnableProvenance { model_count: *n }), path, usage_kind, } @@ -1569,6 +2151,22 @@ async fn asset_graph( .collect(); runnables.sort_by(|a, b| a.path.cmp(&b.path)); + // Only between relations this graph actually renders — an edge to a node + // that is not here has nothing to draw. + let rendered: std::collections::HashSet<&str> = + assets.iter().map(|a| a.path.as_str()).collect(); + let mut dbt_edges: Vec = dbt_edge_rows + .into_iter() + .filter(|r| { + r.from_path != r.to_path + && rendered.contains(r.from_path.as_str()) + && rendered.contains(r.to_path.as_str()) + }) + .map(|r| DbtLineageEdge { from_asset_path: r.from_path, to_asset_path: r.to_path }) + .collect(); + dbt_edges.sort(); + dbt_edges.dedup(); + Ok(Json(AssetGraphResponse { assets, runnables, @@ -1576,6 +2174,9 @@ async fn asset_graph( triggers, macro_edges, test_edges, + dbt_edges, + dbt_snapshot_job, + dbt_graph_ingested_at, })) } diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs new file mode 100644 index 0000000000..82e32797d0 --- /dev/null +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -0,0 +1,444 @@ +//! What a caller who cannot see a dbt script gets from a run of it. +//! +//! The share-link case: `require_job_read_access` has already let them through +//! to the job, so `asset_graph_for` is handed the run — but the project itself +//! is not theirs to read. The graph's SHAPE must survive that and its model SQL +//! must not, and the two are decided by different predicates. Resolving the +//! version from the caller's `script` access instead of from the job emptied the +//! whole dbt half, and every later fix in this area re-touched one of the two. + +use sqlx::{Pool, Postgres}; +use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun}; +use windmill_api_auth::ApiAuthed; +use windmill_common::db::UserDB; + +const WS: &str = "test-workspace"; +const PATH: &str = "f/private/proj"; +const HASH: i64 = 42; + +/// A member of the workspace with no grant on `f/private` — the reason someone +/// is sent a share link in the first place. +fn outsider() -> ApiAuthed { + ApiAuthed { + email: "outsider@windmill.dev".to_string(), + username: "outsider".to_string(), + is_admin: false, + is_operator: false, + groups: vec![], + folders: vec![], + scopes: None, + username_override: None, + username_override_is_token_label: false, + is_session_token: false, + token_prefix: None, + read_only: false, + } +} + +async fn seed(db: &Pool, job: uuid::Uuid) { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) + VALUES ($1, 'private', 'private', '{}', '{}')", + WS + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + HASH, + PATH, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest) + VALUES ($1, $2, $3, $4, 'd')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, raw_code, tags, description, + columns, freshness) + VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders', + 'u/a/wh/analytics/orders', 'select 1', '{finance}', 'daily order facts', + '{"order_id": {"description": "natural key"}}'::jsonb, + '{"warn_after": {"count": 12, "period": "hour"}}'::jsonb)"#, + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + // A test node, for the arguments it carries: `accepted_values` spells out a + // column's domain. + sqlx::query!( + r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, tags, test_kind, test_column, test_args, + attached_node) + VALUES ($1, $2, $3, $4, 'test.p.accepted_values_orders_status', 'test', + 'accepted_values_orders_status', '{}', 'accepted_values', 'status', + '{"values": ["gold", "silver"]}'::jsonb, 'model.p.orders')"#, + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); +} + +/// Everything on a node that the project's author wrote, rather than the shape +/// of the relation it produces. +const AUTHORED: [&str; 5] = [ + "select 1", + "daily order facts", + "finance", + "natural key", + "gold", +]; + +fn query() -> GraphQuery { + GraphQuery { asset_kinds: Some("dbt".to_string()), folder: None, dbt_script_hash: None } +} + +/// Pinned to a run they are entitled to, the outsider gets the model — and +/// nothing its author wrote. Both halves matter: dropping the first is the blank +/// Models panel under working progress rows, dropping the second hands the +/// project's source and documentation to anyone holding a link. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_pinned_run_survives_no_access_to_its_script(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + + let pinned = PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + let res = asset_graph_for( + &outsider(), + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(pinned), + ) + .await + .unwrap(); + let body = serde_json::to_value(&res.0).unwrap(); + + let nodes = body["dbt_nodes"] + .as_array() + .or(body["assets"].as_array()) + .unwrap(); + assert!( + !nodes.is_empty(), + "a run the caller may read must render, whatever their access to the project: {body}" + ); + assert_eq!( + body["dbt_snapshot_job"], + serde_json::json!(job), + "and the marker must agree with it, or the page polls the graph 40 times" + ); + for authored in AUTHORED { + assert!( + !body.to_string().contains(authored), + "`{authored}` is the project's, and stays behind access to it: {body}" + ); + } + assert!( + body.to_string().contains("u/a/wh/analytics/orders"), + "while the relation the run wrote is what the page is for: {body}" + ); + + // The same read by someone who may open the project: the gate has to be the + // caller's access, not a field this endpoint stopped serving. + let seen = asset_graph_for( + &ApiAuthed { is_admin: true, ..outsider() }, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }), + ) + .await + .unwrap(); + let seen = serde_json::to_value(&seen.0).unwrap().to_string(); + for authored in AUTHORED { + assert!( + seen.contains(authored), + "`{authored}` renders for a reader of the project: {seen}" + ); + } +} + +/// Unpinned, the same caller sees nothing of the project: the workspace graph +/// answers for their own access, and this one is not theirs. This is the half +/// that must NOT be relaxed by making the pinned case work. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn without_a_run_the_same_caller_sees_no_dbt_nodes(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + + let res = asset_graph_for( + &outsider(), + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + None, + ) + .await + .unwrap(); + let body = serde_json::to_value(&res.0).unwrap(); + assert!( + !body.to_string().contains("orders"), + "an unpinned graph is the caller's own view of the workspace: {body}" + ); +} + +/// Archiving retires the script; it must not retire the runs that already +/// happened. The pinned read resolves versions through a CTE that skips archived +/// rows, so nothing about the CURRENT workspace graph depends on the sidecar +/// surviving — which is exactly why deleting it looks safe and is not. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn archiving_the_script_leaves_its_finished_runs_renderable(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + sqlx::query!( + "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + WS, + HASH + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + let res = asset_graph_for( + &admin, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(pinned), + ) + .await + .unwrap(); + assert!( + serde_json::to_value(&res.0) + .unwrap() + .to_string() + .contains("orders"), + "a completed run of an archived version still renders its models" + ); + + let now = asset_graph_for( + &admin, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + None, + ) + .await + .unwrap(); + assert!( + !serde_json::to_value(&now.0) + .unwrap() + .to_string() + .contains("model.p.orders"), + "while the workspace graph, which describes what is live, drops it" + ); +} + +/// `extra_perms` is a grant on a ROW, so a path recreated with narrower ones +/// leaves the old version readable to whoever the old row named. The source +/// probe therefore has to name the version it is about to return: matched on +/// the path alone, that stale grant answered for the pinned version's SQL and +/// handed a viewer the body of a project they were never given. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_old_grant_at_the_same_path_does_not_expose_a_newer_version(db: Pool) { + const V2: i64 = 43; + let v1_job = uuid::Uuid::from_u128(7); + let v2_job = uuid::Uuid::from_u128(8); + seed(&db, v1_job).await; + + // The version the outsider WAS granted, archived — the shape the grant + // outlives the deploy in. + sqlx::query!( + r#"UPDATE script SET archived = true, extra_perms = '{"u/outsider": true}'::jsonb + WHERE workspace_id = $1 AND hash = $2"#, + WS, + HASH + ) + .execute(&db) + .await + .unwrap(); + + // The version they were not. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + V2, + PATH, + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest) + VALUES ($1, $2, $3, $4, 'd2')", + WS, + PATH, + V2, + v2_job + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, raw_code, tags) + VALUES ($1, $2, $3, $4, 'model.p.orders', 'model', 'orders', + 'u/a/wh/analytics/orders', 'select 2', '{}')", + WS, + PATH, + V2, + v2_job + ) + .execute(&db) + .await + .unwrap(); + + let pinned = PinnedRun { job_id: v2_job, script_path: PATH.to_string(), script_hash: Some(V2) }; + let res = asset_graph_for( + &outsider(), + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(pinned), + ) + .await + .unwrap(); + let body = serde_json::to_value(&res.0).unwrap().to_string(); + + assert!( + body.contains("orders"), + "the run they were given still renders its models: {body}" + ); + assert!( + !body.contains("select 2"), + "but not the SQL of a version their grant never covered: {body}" + ); +} + +/// The editor's own graph, keyed to the parse job and to no version. +async fn seed_editor_graph(db: &Pool, job: uuid::Uuid) { + sqlx::query!( + "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest) + VALUES ($1, $2, NULL, $3, 'd')", + WS, + PATH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, raw_code, tags) + VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft', + 'u/a/wh/analytics/draft', 'select 3', '{}')"#, + WS, + PATH, + job + ) + .execute(db) + .await + .unwrap(); +} + +/// A buffer parse is the third provenance: no deployed version behind it, and +/// its models are ones no `asset` row has heard of. It renders when pinned to +/// the job that produced it — the only way in — and its SQL comes with it, since +/// it exists because this caller's own parse job wrote the buffer. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_editor_graph_renders_only_through_its_own_job(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + let parse = uuid::Uuid::from_u128(9); + seed_editor_graph(&db, parse).await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = asset_graph_for( + &admin, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(PinnedRun { job_id: parse, script_path: PATH.to_string(), script_hash: None }), + ) + .await + .unwrap(); + let body = serde_json::to_value(&pinned.0).unwrap(); + assert!( + body.to_string().contains("u/a/wh/analytics/draft"), + "the buffer's own models render: {body}" + ); + assert!( + body.to_string().contains("select 3"), + "and their SQL, which is what the caller just wrote: {body}" + ); + assert_eq!( + body["dbt_snapshot_job"], + serde_json::json!(parse), + "labelled as a graph of its own, so the editor can say where it came from" + ); + assert!( + !body.to_string().contains("select 1"), + "and the deployed version's models are not mixed into it: {body}" + ); + + // Through the PATH — which is what the workspace graph and every run of the + // deployed version ask for — a buffer parse must not appear at all. It + // describes an editor's unsaved state, not what the script owns. + let workspace = asset_graph_for(&admin, WS, UserDB::new(db.clone()), db.clone(), query(), None) + .await + .unwrap(); + let workspace = serde_json::to_value(&workspace.0).unwrap().to_string(); + assert!( + !workspace.contains("u/a/wh/analytics/draft"), + "an editor's buffer is not part of the workspace graph: {workspace}" + ); + + let deployed_run = asset_graph_for( + &admin, + WS, + UserDB::new(db.clone()), + db.clone(), + query(), + Some(PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }), + ) + .await + .unwrap(); + let deployed_run = serde_json::to_value(&deployed_run.0).unwrap().to_string(); + assert!( + !deployed_run.contains("u/a/wh/analytics/draft"), + "nor of a run of the deployed version: {deployed_run}" + ); +} diff --git a/backend/windmill-api-assets/tests/fixtures/base.sql b/backend/windmill-api-assets/tests/fixtures/base.sql new file mode 100644 index 0000000000..412fa1029f --- /dev/null +++ b/backend/windmill-api-assets/tests/fixtures/base.sql @@ -0,0 +1,146 @@ +-- used for backend automated testing +-- https://docs.rs/sqlx/latest/sqlx/attr.test.html + +INSERT INTO workspace + (id, name, owner) + VALUES ('test-workspace', 'test-workspace', 'test-user'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test@windmill.dev', 'test-user', true, 'Admin'); + +INSERT INTO workspace_key(workspace_id, kind, key) VALUES + ('test-workspace', 'cloud', 'test-key'); + + +INSERT INTO workspace_settings (workspace_id) VALUES + ('test-workspace'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('test@windmill.dev', 'not-a-real-hash', 'password', true, true, 'Test User', 'test-user'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('test2@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 2'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('test3@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 3'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test2@windmill.dev', 'test-user-2', false, 'User'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test3@windmill.dev', 'test-user-3', false, 'User'); + +insert INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('SECRET_TOKEN'::bytea), 'hex'), 'SECRET_TOK', 'SECRET_TOKEN', 'test@windmill.dev', 'test token', true); +insert INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('SECRET_TOKEN_2'::bytea), 'hex'), 'SECRET_TOK', 'SECRET_TOKEN_2', 'test2@windmill.dev', 'test token 2', false); +insert INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('SECRET_TOKEN_3'::bytea), 'hex'), 'SECRET_TOK', 'SECRET_TOKEN_3', 'test3@windmill.dev', 'test token 3', false); + +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin; +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; + +CREATE FUNCTION "notify_insert_on_completed_job" () +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('completed', NEW.id::text); + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + CREATE TRIGGER "notify_insert_on_completed_job" + AFTER INSERT ON "v2_job_completed" + FOR EACH ROW +EXECUTE FUNCTION "notify_insert_on_completed_job" (); + + +CREATE FUNCTION "notify_queue" () +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('queued', NEW.id::text); + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + CREATE TRIGGER "notify_queue_after_insert" + AFTER INSERT ON "v2_job_queue" + FOR EACH ROW +EXECUTE FUNCTION "notify_queue" (); + + CREATE TRIGGER "notify_queue_after_flow_status_update" + AFTER UPDATE ON "v2_job_status" + FOR EACH ROW + WHEN (NEW.flow_status IS DISTINCT FROM OLD.flow_status) +EXECUTE FUNCTION "notify_queue" (); + +-- Apply phase 4: +DROP FUNCTION IF EXISTS v2_job_after_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_completed_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_completed_before_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_queue_after_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_queue_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_queue_before_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_runtime_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_runtime_before_update CASCADE; +DROP FUNCTION IF EXISTS v2_job_status_before_insert CASCADE; +DROP FUNCTION IF EXISTS v2_job_status_before_update CASCADE; + +DROP VIEW IF EXISTS completed_job, completed_job_view, job, queue, queue_view CASCADE; + +ALTER TABLE v2_job_queue + DROP COLUMN IF EXISTS __parent_job CASCADE, + DROP COLUMN IF EXISTS __created_by CASCADE, + DROP COLUMN IF EXISTS __script_hash CASCADE, + DROP COLUMN IF EXISTS __script_path CASCADE, + DROP COLUMN IF EXISTS __args CASCADE, + DROP COLUMN IF EXISTS __logs CASCADE, + DROP COLUMN IF EXISTS __raw_code CASCADE, + DROP COLUMN IF EXISTS __canceled CASCADE, + DROP COLUMN IF EXISTS __last_ping CASCADE, + DROP COLUMN IF EXISTS __job_kind CASCADE, + DROP COLUMN IF EXISTS __env_id CASCADE, + DROP COLUMN IF EXISTS __schedule_path CASCADE, + DROP COLUMN IF EXISTS __permissioned_as CASCADE, + DROP COLUMN IF EXISTS __flow_status CASCADE, + DROP COLUMN IF EXISTS __raw_flow CASCADE, + DROP COLUMN IF EXISTS __is_flow_step CASCADE, + DROP COLUMN IF EXISTS __language CASCADE, + DROP COLUMN IF EXISTS __same_worker CASCADE, + DROP COLUMN IF EXISTS __raw_lock CASCADE, + DROP COLUMN IF EXISTS __pre_run_error CASCADE, + DROP COLUMN IF EXISTS __email CASCADE, + DROP COLUMN IF EXISTS __visible_to_owner CASCADE, + DROP COLUMN IF EXISTS __mem_peak CASCADE, + DROP COLUMN IF EXISTS __root_job CASCADE, + DROP COLUMN IF EXISTS __leaf_jobs CASCADE, + DROP COLUMN IF EXISTS __concurrent_limit CASCADE, + DROP COLUMN IF EXISTS __concurrency_time_window_s CASCADE, + DROP COLUMN IF EXISTS __timeout CASCADE, + DROP COLUMN IF EXISTS __flow_step_id CASCADE, + DROP COLUMN IF EXISTS __cache_ttl CASCADE; + +LOCK TABLE v2_job_queue IN ACCESS EXCLUSIVE MODE; +ALTER TABLE v2_job_completed + DROP COLUMN IF EXISTS __parent_job CASCADE, + DROP COLUMN IF EXISTS __created_by CASCADE, + DROP COLUMN IF EXISTS __created_at CASCADE, + DROP COLUMN IF EXISTS __success CASCADE, + DROP COLUMN IF EXISTS __script_hash CASCADE, + DROP COLUMN IF EXISTS __script_path CASCADE, + DROP COLUMN IF EXISTS __args CASCADE, + DROP COLUMN IF EXISTS __logs CASCADE, + DROP COLUMN IF EXISTS __raw_code CASCADE, + DROP COLUMN IF EXISTS __canceled CASCADE, + DROP COLUMN IF EXISTS __job_kind CASCADE, + DROP COLUMN IF EXISTS __env_id CASCADE, + DROP COLUMN IF EXISTS __schedule_path CASCADE, + DROP COLUMN IF EXISTS __permissioned_as CASCADE, + DROP COLUMN IF EXISTS __raw_flow CASCADE, + DROP COLUMN IF EXISTS __is_flow_step CASCADE, + DROP COLUMN IF EXISTS __language CASCADE, + DROP COLUMN IF EXISTS __is_skipped CASCADE, + DROP COLUMN IF EXISTS __raw_lock CASCADE, + DROP COLUMN IF EXISTS __email CASCADE, + DROP COLUMN IF EXISTS __visible_to_owner CASCADE, + DROP COLUMN IF EXISTS __tag CASCADE, + DROP COLUMN IF EXISTS __priority CASCADE; diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 5219b41e9b..b3ad811b7f 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -26,7 +26,8 @@ use windmill_common::DB; use windmill_common::{ auth::{ - get_folders_for_user, get_groups_for_user, hash_token, safe_token_prefix, JWTAuthClaims, + get_folders_for_user, get_groups_for_user, hash_token, is_session_label, safe_token_prefix, + JWTAuthClaims, }, error::{Error, JsonResult}, jwt, @@ -196,7 +197,9 @@ impl AuthCache { tracing::error!("JWT auth error: workspace_id mismatch"); return None; } - let username_override = username_override_from_label(claims.label); + let is_session_token = is_session_label(claims.label.as_deref()); + let (username_override, username_override_is_token_label) = + username_override_from_label(claims.label); let authed = ApiAuthed { email: claims.email, @@ -211,6 +214,8 @@ impl AuthCache { // WM_TOKEN) keeps full user privileges as before. scopes: claims.scopes, username_override, + username_override_is_token_label, + is_session_token, token_prefix: claims.audit_span, read_only: false, }; @@ -265,7 +270,9 @@ impl AuthCache { (Some(owner), Some(email), super_admin, _, label, read_only) if w_id.is_some() => { - let username_override = username_override_from_label(label); + let is_session_token = is_session_label(label.as_deref()); + let (username_override, username_override_is_token_label) = + username_override_from_label(label); if let Some((prefix, name)) = owner.split_once('/') { if prefix == "u" { let lookup = if super_admin { @@ -308,6 +315,8 @@ impl AuthCache { folders, scopes: None, username_override, + username_override_is_token_label, + is_session_token, token_prefix: Some(safe_token_prefix(token)), read_only, }) @@ -358,6 +367,8 @@ impl AuthCache { folders, scopes: None, username_override, + username_override_is_token_label, + is_session_token, token_prefix: Some(safe_token_prefix(token)), read_only, }) @@ -386,7 +397,9 @@ impl AuthCache { } } (_, Some(email), super_admin, scopes, label, read_only) => { - let username_override = username_override_from_label(label); + let is_session_token = is_session_label(label.as_deref()); + let (username_override, username_override_is_token_label) = + username_override_from_label(label); if w_id.is_some() { let row_o = sqlx::query!( "SELECT username, is_admin, operator FROM usr WHERE @@ -429,6 +442,8 @@ impl AuthCache { folders, scopes, username_override, + username_override_is_token_label, + is_session_token, token_prefix: Some(safe_token_prefix(token)), read_only, }) @@ -450,6 +465,8 @@ impl AuthCache { folders: vec![], scopes, username_override, + username_override_is_token_label, + is_session_token, token_prefix: Some(safe_token_prefix(token)), read_only, }), @@ -473,6 +490,8 @@ impl AuthCache { folders: Vec::new(), scopes, username_override, + username_override_is_token_label, + is_session_token, token_prefix: Some(safe_token_prefix(token)), read_only, }) @@ -508,6 +527,8 @@ impl AuthCache { folders: Vec::new(), scopes: None, username_override: None, + username_override_is_token_label: false, + is_session_token: false, token_prefix: Some(safe_token_prefix(token)), read_only: false, }; @@ -715,6 +736,8 @@ fn no_auth_admin_authed() -> ApiAuthed { folders: Vec::new(), scopes: None, username_override: None, + username_override_is_token_label: false, + is_session_token: false, token_prefix: None, read_only: false, } @@ -835,27 +858,47 @@ pub async fn resolve_opt_job_authed( Err((Error::NotAuthorized("Unauthorized".to_string()), parts)) } -fn username_override_from_label(label: Option) -> Option { +/// Returns the override and whether it names the token's *label* rather than the entity that +/// fired the request. Callers must not re-derive the second element from the first: the +/// `ephemeral-script-end-user-` arm forwards a `created_by` verbatim, and `created_by` is +/// unconstrained, so it may itself look like any of these shapes. +/// +/// Only namespaces `create_token` rejects (`is_server_minted_label`) are trusted to name the +/// entity acting, so the label can only have come from a server-side mint. Tokens minted +/// before that guard existed are the remaining hole; closing it needs the token row to record +/// who minted it rather than inferring it from the label. +/// +/// Note that a trigger whose identity is set server-side — the SMTP one builds an `email-*` +/// override directly — does not rely on this at all, so its prefix must not be trusted here. +pub(crate) fn username_override_from_label(label: Option) -> (Option, bool) { match label { + Some(label) if label.starts_with("ephemeral-webhook-") => (Some(label), false), + Some(label) if label.starts_with("ephemeral-script-end-user-") => ( + Some( + label + .trim_start_matches("ephemeral-script-end-user-") + .to_string(), + ), + false, + ), + // User-mintable, so they name nobody in particular — the trigger panels merely + // pre-fill `webhook-`/`http-`, and the editor mints the lsp one. The override keeps + // its value because `require_job_read_access` matches it against the `created_by` of + // jobs launched under it, which these shapes produced while they were trusted. + Some(label) if label == "Ephemeral lsp token" => (Some("lsp".to_string()), true), Some(label) - if label.starts_with("ephemeral-webhook-") - || label.starts_with("webhook-") + if label.starts_with("webhook-") || label.starts_with("http-") || label.starts_with("email-") || label.starts_with("ws-") => { - Some(label) + (Some(label), true) } - Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( - label - .trim_start_matches("ephemeral-script-end-user-") - .to_string(), + Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => ( + Some(format!("{}{label}", crate::GENERIC_TOKEN_LABEL_PREFIX)), + true, ), - Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), - Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { - Some(format!("label-{label}")) - } - _ => None, + _ => (None, false), } } diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 8120c16605..d92c3a5f2d 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -23,6 +23,8 @@ use windmill_common::{ }, db::{Authable, Authed, AuthedRef}, error::{self, Error, Result}, + jobs::JobTriggerKind, + triggers::TriggerMetadata, users::username_to_permissioned_as, DB, }; @@ -37,6 +39,11 @@ pub use auth::{ // ------------ ApiAuthed & OptJobAuthed types ------------ +/// Prefix `username_override_from_label` puts on the label of a generic user token. The +/// override keeps this form even though `display_username` skips it: `require_job_read_access` +/// matches it against `created_by` to let a token re-read the jobs it launched. +pub const GENERIC_TOKEN_LABEL_PREFIX: &str = "label-"; + #[derive(Default, Clone, Debug)] pub struct OptJobAuthed { pub job_id: Option, @@ -54,6 +61,15 @@ pub struct ApiAuthed { pub folders: Vec<(String, bool, bool)>, pub scopes: Option>, pub username_override: Option, + /// Whether `username_override` is a generic user-token label rather than a name that + /// identifies the requester. It cannot be recovered from the value: the ephemeral + /// end-user override passes a `created_by` through verbatim, and that may itself be a + /// `label-*` string. Only `username_override_from_label` sets it. + pub username_override_is_token_label: bool, + /// Whether the request authenticated with the session token minted at browser login. + /// Only `trigger_or_fallback` reads it — see `is_session_label` for why it attributes + /// rather than proves, and must not gate authority. + pub is_session_token: bool, pub token_prefix: Option, pub read_only: bool, } @@ -72,8 +88,43 @@ impl ApiAuthed { } } + /// The name a run triggered by this principal is credited to (`v2_job.created_by`). A + /// trigger-token override names the entity that fired the request and wins; a generic + /// token label does not, so the token owner is credited and stays traceable even when + /// `permissioned_as` is an on-behalf-of identity. The audit `end_user` is the override + /// itself, label included, so the two diverge for a labeled token. pub fn display_username(&self) -> &str { - self.username_override.as_ref().unwrap_or(&self.username) + match self.username_override.as_deref() { + Some(o) if !self.username_override_is_token_label => o, + _ => &self.username, + } + } + + /// Set an override that names the entity acting, e.g. a trigger. Assigning + /// `username_override` on its own would keep the provenance flag of whatever this authed + /// was built from, and a stale `true` makes `display_username` ignore the new value. + pub fn set_acting_username_override(&mut self, username_override: Option) { + self.username_override = username_override; + self.username_override_is_token_label = false; + } + + /// The `trigger_kind` a run started through a `/jobs/run*` route is stamped with: a trigger + /// that built its own metadata always wins, and a run driven by any other token — webhooks, + /// the CLI, the SDKs — is `webhook`, matching the `wm_trigger.kind` the preprocessor already + /// reports for these routes. Derived from the token, never from the request, because the + /// column is authority-bearing for other kinds (`app` marks a file as app-produced). + /// + /// A browser session is left unstamped rather than marked [`JobTriggerKind::Ui`]: that label + /// is one a worker built before this release cannot decode, and it would strand the jobs + /// carrying it. `webhook` has always been decodable, so it is safe to write today. + pub fn trigger_or_fallback(&self, trigger: Option) -> Option { + if trigger.is_some() { + return trigger; + } + if self.is_session_token { + return None; + } + Some(TriggerMetadata::new(None, JobTriggerKind::Webhook)) } } @@ -103,6 +154,8 @@ impl From for ApiAuthed { folders: value.folders, scopes: value.scopes, username_override: None, + username_override_is_token_label: false, + is_session_token: false, token_prefix: value.token_prefix, read_only: false, } @@ -414,6 +467,9 @@ fn scope_contains(caller: &ScopeDefinition, requested: &ScopeDefinition) -> bool // write subsumes read; otherwise the action must match exactly. match (caller.action.as_str(), requested.action.as_str()) { (c, r) if c == r || (c == "write" && r == "read") => {} + // Apps only: `write` covers `run` (see `ScopeDefinition::includes`), so an + // app-editor token can mint the narrower run-only credential. + ("write", "run") if caller.domain == "apps" => {} _ => return false, } @@ -852,6 +908,8 @@ pub async fn fetch_api_authed_from_permissioned_as( folders: authed.folders, scopes: authed.scopes, username_override: None, + username_override_is_token_label: false, + is_session_token: false, token_prefix: authed.token_prefix, read_only: false, }; @@ -869,7 +927,8 @@ pub async fn fetch_api_authed_from_permissioned_as( } }; - api_authed.username_override = username_override; + // Callers pass a trigger or app identity here, never a token label. + api_authed.set_acting_username_override(username_override); Ok(api_authed) } @@ -1194,6 +1253,114 @@ mod tests { } } + /// `display_username` is what `push` credits a run to, so a token label standing in for + /// it erases the caller from `created_by` and from the audit trail — irrecoverably when + /// `permissioned_as` is an on-behalf-of identity that also takes the `username` slot. + #[test] + fn generic_token_label_credits_the_token_owner() { + let owner_of = |label: &str| { + let (username_override, username_override_is_token_label) = + auth::username_override_from_label(Some(label.to_string())); + ApiAuthed { + username: "alice".into(), + username_override, + username_override_is_token_label, + ..Default::default() + } + }; + + // Arbitrary user-chosen labels, and the auto-generated MCP OAuth one. + assert_eq!(owner_of("my-personal-token").display_username(), "alice"); + assert_eq!( + owner_of("mcp-oauth-mcp-client-9f3a1c").display_username(), + "alice" + ); + + // A trigger-*shaped* label is just as user-settable as any other, so it is credited + // the same way. Its value is still kept as the override, for `require_job_read_access`. + let webhookish = owner_of("webhook-f/svc/my_script"); + assert_eq!(webhookish.display_username(), "alice"); + assert_eq!( + webhookish.username_override.as_deref(), + Some("webhook-f/svc/my_script") + ); + + // Only labels `create_token` refuses to mint name the entity that fired the request. + assert_eq!( + owner_of("ephemeral-webhook-google-abc12").display_username(), + "ephemeral-webhook-google-abc12" + ); + + // Minted by the editor through the public handler, so it names no principal either. + assert_eq!(owner_of("Ephemeral lsp token").display_username(), "alice"); + + // The SMTP trigger sets its `email-*` identity server-side rather than through a + // label, so a token carrying that prefix is just a user token. + assert_eq!(owner_of("email-f/team/inbox").display_username(), "alice"); + assert_eq!( + owner_of("ephemeral-script-end-user-enduser42").display_username(), + "enduser42" + ); + + // The end-user token forwards a `created_by` verbatim, and `created_by` is not + // constrained to a username — a job launched before the owner was credited still + // carries `label-*`. That is an end user, not this token's label, so it stands. + assert_eq!( + owner_of("ephemeral-script-end-user-label-alice").display_username(), + "label-alice" + ); + } + + /// A browser session is the one shape left unstamped, so the Runs page can say "a token + /// started this" without claiming the converse. Every other label — every shape a member + /// can pass to `create_token` — is `webhook`. + #[test] + fn only_a_browser_session_is_left_unstamped() { + let kind_of = |label: Option<&str>| { + ApiAuthed { + is_session_token: windmill_common::auth::is_session_label(label), + ..Default::default() + } + .trigger_or_fallback(None) + .map(|t| t.trigger_kind.to_string()) + }; + + assert_eq!(kind_of(Some("session")), None); + + for label in [ + Some("my-personal-token"), + Some("webhook-f/svc/my_script"), + Some("Ephemeral lsp token"), + Some("ephemeral-script"), + Some("ephemeral-webhook-google-abc12"), + Some("mcp-oauth-mcp-client-9f3a1c"), + Some(""), + // A label-less token: the job WM_TOKEN, and any token created without one. + None, + ] { + assert_eq!( + kind_of(label).as_deref(), + Some("webhook"), + "label {label:?}" + ); + } + } + + /// A trigger that built its own metadata must survive the fallback, or a scheduled or + /// routed run started under a personal token would be re-attributed to a webhook. + #[test] + fn a_real_trigger_wins_over_the_token_fallback() { + let authed = ApiAuthed::default(); + let schedule = TriggerMetadata::new( + Some("u/alice/nightly".to_string()), + JobTriggerKind::Schedule, + ); + + let kept = authed.trigger_or_fallback(Some(schedule)).unwrap(); + assert_eq!(kept.trigger_kind.to_string(), "schedule"); + assert_eq!(kept.trigger_path.as_deref(), Some("u/alice/nightly")); + } + // Regression tests for the Preview path traversal: a Preview's path skips the // DB `proper_id` CHECK and reaches the worker, where it builds on-disk module // dirs. Traversal must be rejected even for admins, who otherwise bypass the @@ -1484,6 +1651,25 @@ mod tests { opt_scopes(Some(vec!["users:read", "if_jobs:filter_tags:default"])).as_deref() ) .is_ok()); + // Apps `write` covers `run`, so an app-editor token can mint the run-only + // credential for the same app — but only within its own resource subtree, + // and the equivalence stays Apps-only. + let app_editor = authed_with_scopes(Some(vec!["apps:write:u/me/a", "jobs:write"])); + assert!(ensure_scopes_within_caller( + &app_editor, + opt_scopes(Some(vec!["apps:run:u/me/a"])).as_deref() + ) + .is_ok()); + assert!(ensure_scopes_within_caller( + &app_editor, + opt_scopes(Some(vec!["apps:run:u/me/b"])).as_deref() + ) + .is_err()); + assert!(ensure_scopes_within_caller( + &app_editor, + opt_scopes(Some(vec!["jobs:run"])).as_deref() + ) + .is_err()); } #[test] diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 75cbc104ac..4738fec572 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -120,6 +120,10 @@ impl ScopeDefinition { match (self.action.as_str(), other.action.as_str()) { (a, b) if (a == "write" && b == "read") || (a == b) => {} + // Apps only: `write` can rewrite the app and its policy, so it also covers + // running its components. Not general — `jobs:write` must not grant + // `jobs:run`. The resource check below still confines it to the same app. + ("write", "run") if self.domain == "apps" => {} _ => return false, } @@ -802,6 +806,15 @@ fn scope_grants_access( && route_path.is_some_and(resource_metadata_route_allowed)); } + // Apps `write` covers `run` (see `ScopeDefinition::includes`). Like every domain + // here this layer is resource-blind; the Run handlers path-check the app. + if scope_domain == ScopeDomain::Apps + && scope_action == ScopeAction::Write + && required_action == ScopeAction::Run + { + return Ok(true); + } + if !scope_action.includes(&required_action) && !(scope_domain == ScopeDomain::Jobs && required_action == ScopeAction::Read @@ -1016,6 +1029,29 @@ mod tests { assert!(check_route_access(&sc, "/api/w/test/data_metrics/list", "GET").is_err()); } + /// `apps_u/execute_component` (and the S3 upload the same components drive) is a + /// Run action, so a scoped token needs `apps:run`. `apps:write` must keep reaching + /// it too: it can rewrite the app and its policy, so withholding execution from it + /// protects nothing while breaking every app-scoped token. + #[test] + fn apps_run_routes_accept_run_and_write_scopes() { + let execute = "/api/w/test/apps_u/execute_component/u/admin/app"; + for scope in ["apps:run", "apps:write"] { + assert!( + check_route_access(&[scope.to_string()], execute, "POST").is_ok(), + "{scope} must reach execute_component" + ); + } + assert!(check_route_access(&["apps:read".to_string()], execute, "POST").is_err()); + // The write-satisfies-run allowance is confined to the apps domain. + assert!(check_route_access( + &["jobs:write".to_string()], + "/api/w/test/jobs/run/p/u/admin/script", + "POST" + ) + .is_err()); + } + #[test] fn test_new_domain_parsing() { // Test that new domains are properly parsed diff --git a/backend/windmill-api-flows/Cargo.toml b/backend/windmill-api-flows/Cargo.toml index 231c00a159..3fc2c7b572 100644 --- a/backend/windmill-api-flows/Cargo.toml +++ b/backend/windmill-api-flows/Cargo.toml @@ -10,8 +10,9 @@ path = "src/lib.rs" [features] default = [] -enterprise = ["windmill-common/enterprise"] +enterprise = ["windmill-common/enterprise", "windmill-native-triggers?/enterprise"] private = ["windmill-common/private", "windmill-dep-map/private"] +native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger"] [dependencies] windmill-common = { workspace = true, default-features = false } windmill-api-auth.workspace = true @@ -19,6 +20,7 @@ windmill-queue.workspace = true windmill-audit.workspace = true windmill-git-sync.workspace = true windmill-dep-map.workspace = true +windmill-native-triggers = { workspace = true, optional = true } axum.workspace = true hyper.workspace = true diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index ca3d65b2ce..f567f3ae27 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -50,6 +50,7 @@ use windmill_common::{ flows::{EditFlow, Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow}, jobs::JobPayload, schedule::Schedule, + triggers::MovedNativeTrigger, utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, }; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; @@ -577,17 +578,17 @@ async fn create_flow( // Apply folder default_permissioned_as on create when the caller did not // explicitly preserve a value and the user can preserve. - let explicit_preserve = nf.on_behalf_of_email.is_some() + let explicit_preserve = (nf.on_behalf_of_email.is_some() + || nf.on_behalf_of.is_some()) && nf.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed); if !explicit_preserve && windmill_common::can_preserve_on_behalf_of(&authed) { - if let Some(default_email) = - windmill_common::folders::resolve_folder_default_on_behalf_of_email( - &db, &w_id, &nf.path, - ) - .await? + if let Some((default_email, default_permissioned_as)) = + windmill_common::folders::resolve_folder_default_on_behalf_of(&db, &w_id, &nf.path) + .await? { nf.on_behalf_of_email = Some(default_email); + nf.on_behalf_of = Some(default_permissioned_as); nf.preserve_on_behalf_of = Some(true); } } @@ -598,19 +599,35 @@ async fn create_flow( check_schedule_conflict(&mut tx, &w_id, &nf.path).await?; let schema_str = nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()); + let resolved_on_behalf_of = + windmill_common::resolve_on_behalf_of( + nf.on_behalf_of_email.as_deref(), + nf.on_behalf_of.as_deref(), + nf.preserve_on_behalf_of.unwrap_or(false), + &authed, + &w_id, + &db, + ) + .await?; + // Written beside the principal only while a worker that still reads it may be live. + let legacy_on_behalf_of_email = + windmill_common::legacy_on_behalf_of_email(resolved_on_behalf_of.as_deref(), &w_id, &db) + .await?; sqlx::query!( r#"INSERT INTO flow ( workspace_id, path, summary, description, dependency_job, lock_error_logs, tag, - dedicated_worker, visible_to_runner_only, on_behalf_of_email, + dedicated_worker, visible_to_runner_only, ws_error_handler_muted, - value, schema, edited_by, edited_at, labels + value, schema, edited_by, edited_at, labels, + on_behalf_of, on_behalf_of_email ) VALUES ( $1, $2, $3, $4, NULL, '', $5, - $6, $7, $8, - $9, - $10, $11::text::json, $12, now(), $13 + $6, $7, + $8, + $9, $10::text::json, $11, now(), $12, + $13, $14 )"#, w_id, nf.path, @@ -619,16 +636,13 @@ async fn create_flow( nf.tag, nf.dedicated_worker, nf.visible_to_runner_only.unwrap_or(false), - windmill_common::resolve_on_behalf_of_email( - nf.on_behalf_of_email.as_deref(), - nf.preserve_on_behalf_of.unwrap_or(false), - &authed, - ), nf.ws_error_handler_muted.unwrap_or(false), sqlx::types::Json(&nf.value) as _, schema_str, &authed.username, nf.labels.as_deref() as Option<&[String]>, + resolved_on_behalf_of, + legacy_on_behalf_of_email, ) .execute(&mut *tx) .await?; @@ -684,10 +698,10 @@ async fn create_flow( ) .await?; if let Some(on_behalf_of) = windmill_common::check_on_behalf_of_preservation( - nf.on_behalf_of_email.as_deref(), + resolved_on_behalf_of.as_deref(), nf.preserve_on_behalf_of.unwrap_or(false), &authed, - &authed.email, + &windmill_common::users::username_to_permissioned_as(&authed.username), ) { audit_log( &mut *tx, @@ -728,6 +742,7 @@ async fn create_flow( &authed.email, windmill_common::users::username_to_permissioned_as(&authed.username), authed.token_prefix.as_deref(), + authed.username_override.as_deref(), None, None, None, @@ -863,8 +878,27 @@ async fn get_latest_version( Ok(Json(version)) } +/// `on_behalf_of_email` is derived rather than selected: the read paths fill it from the +/// principal so clients written against the address keep working. The column itself still +/// exists for the workers that read it — see `legacy_on_behalf_of_email`. +async fn derived_on_behalf_of_email( + db: &DB, + w_id: &str, + flow: &Flow, +) -> error::Result> { + let Some(permissioned_as) = flow.on_behalf_of.as_deref() else { + return Ok(None); + }; + // Uncached, for the reason given on `prefetch_cached_script`: this pair is round-tripped. + Ok(Some( + windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) + .await?, + )) +} + async fn get_flow_version( authed: ApiAuthed, + Extension(db): Extension, Extension(user_db): Extension, Path((w_id, version, path)): Path<(String, i64, StripPath)>, ) -> JsonResult { @@ -873,26 +907,28 @@ async fn get_flow_version( let mut tx = user_db.begin(&authed).await?; let flow = sqlx::query_as::<_, Flow>( - "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow.labels, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by + "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by FROM flow LEFT JOIN flow_version ON flow_version.path = flow.path AND flow_version.workspace_id = flow.workspace_id WHERE flow.path = $1 AND flow.workspace_id = $2 AND flow_version.id = $3", ) .bind(path) - .bind(w_id) + .bind(&w_id) .bind(version) .fetch_optional(&mut *tx) .await?; tx.commit().await?; - let flow = not_found_if_none(flow, "Flow version", version.to_string())?; + let mut flow = not_found_if_none(flow, "Flow version", version.to_string())?; + flow.on_behalf_of_email = derived_on_behalf_of_email(&db, &w_id, &flow).await?; Ok(Json(flow)) } async fn get_flow_version_by_id( authed: ApiAuthed, + Extension(db): Extension, Extension(user_db): Extension, Path((w_id, version)): Path<(String, i64)>, ) -> JsonResult { @@ -929,7 +965,7 @@ async fn get_flow_version_by_id( flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, - flow.on_behalf_of_email, + flow.on_behalf_of, flow.labels, flow_version.schema, flow_version.value, @@ -948,11 +984,12 @@ async fn get_flow_version_by_id( tx.commit().await?; - let flow = not_found_if_none( + let mut flow = not_found_if_none( flow, "Flow", format!("for version {} (flow may have been deleted)", version), )?; + flow.on_behalf_of_email = derived_on_behalf_of_email(&db, &w_id, &flow).await?; Ok(Json(flow)) } @@ -1002,6 +1039,35 @@ async fn update_flow_history( Ok(()) } +/// Re-point the webhooks of the native triggers a rename carried onto the new path. +/// +/// Runs after the deploy transaction commits — repointing a webhook is not undoable — and off the +/// request, because it waits on a third-party service that may be slow or gone, and a deploy that +/// already committed must not look like it failed. The rename itself marked these rows +/// `REREGISTRATION_PENDING`, so nothing is lost silently if this never finishes. +fn reregister_moved_native_triggers( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + moved: Vec, +) { + if moved.is_empty() { + return; + } + #[cfg(feature = "native_trigger")] + { + let (db, authed, w_id) = (db.clone(), authed.clone(), w_id.to_string()); + tokio::spawn(async move { + windmill_native_triggers::rename::reregister_triggers_after_rename( + &db, &authed, &w_id, &moved, + ) + .await; + }); + } + #[cfg(not(feature = "native_trigger"))] + let _ = (db, authed, w_id, moved); +} + async fn update_flow( authed: ApiAuthed, Extension(user_db): Extension, @@ -1021,6 +1087,13 @@ async fn update_flow( // A `<= 0` flow timeout is "unset", not a 0-second limit (see create_flow). nf.timeout = windmill_common::runnable_settings::none_if_non_positive(nf.timeout); check_scopes(&authed, || format!("flows:write:{}", flow_path))?; + // A rename writes the destination as much as the source, so a path-scoped token needs both. + // Checking only the source would let it move a flow onto a path it has no say over — and + // everything that follows the rename, native triggers included, is then acting on a path this + // caller was never authorized for. `create_script` already scopes against its destination. + if nf.path != flow_path { + check_scopes(&authed, || format!("flows:write:{}", nf.path))?; + } if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, @@ -1053,6 +1126,20 @@ async fn update_flow( let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?; let is_new_path = nf.path != flow_path; let schema_str = schema.and_then(|x| serde_json::to_string(&x).ok()); + let resolved_on_behalf_of = + windmill_common::resolve_on_behalf_of( + nf.on_behalf_of_email.as_deref(), + nf.on_behalf_of.as_deref(), + nf.preserve_on_behalf_of.unwrap_or(false), + &authed, + &w_id, + &db, + ) + .await?; + // Written beside the principal only while a worker that still reads it may be live. + let legacy_on_behalf_of_email = + windmill_common::legacy_on_behalf_of_email(resolved_on_behalf_of.as_deref(), &w_id, &db) + .await?; sqlx::query!( " @@ -1067,26 +1154,22 @@ async fn update_flow( tag = $4, dedicated_worker = $5, visible_to_runner_only = $6, - on_behalf_of_email = $7, - ws_error_handler_muted = $8, - value = $9, - schema = $10::text::json, - edited_by = $11, + ws_error_handler_muted = $7, + value = $8, + schema = $9::text::json, + edited_by = $10, edited_at = now(), - labels = COALESCE($14, labels) + labels = COALESCE($13, labels), + on_behalf_of = $14, + on_behalf_of_email = $15 WHERE - path = $12 AND workspace_id = $13", + path = $11 AND workspace_id = $12", if is_new_path { flow_path } else { &nf.path }, nf.summary, nf.description.as_deref().unwrap_or(""), nf.tag, nf.dedicated_worker, nf.visible_to_runner_only.unwrap_or(false), - windmill_common::resolve_on_behalf_of_email( - nf.on_behalf_of_email.as_deref(), - nf.preserve_on_behalf_of.unwrap_or(false), - &authed, - ), nf.ws_error_handler_muted.unwrap_or(false), sqlx::types::Json(&nf.value) as _, schema_str, @@ -1094,6 +1177,8 @@ async fn update_flow( flow_path, w_id, nf.labels.as_deref() as Option<&[String]>, + resolved_on_behalf_of, + legacy_on_behalf_of_email, ) .execute(&mut *tx) .await @@ -1105,8 +1190,8 @@ async fn update_flow( // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint sqlx::query!( "INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels) - SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels) + SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels FROM flow WHERE path = $2 AND workspace_id = $3", nf.path, @@ -1237,8 +1322,9 @@ async fn update_flow( } } + let mut moved_native_triggers = Vec::new(); if is_new_path { - windmill_common::triggers::update_triggers_script_path( + moved_native_triggers = windmill_common::triggers::update_triggers_script_path( &mut tx, &nf.path, &flow_path, &w_id, true, ) .await @@ -1280,10 +1366,10 @@ async fn update_flow( ) .await?; if let Some(on_behalf_of) = windmill_common::check_on_behalf_of_preservation( - nf.on_behalf_of_email.as_deref(), + resolved_on_behalf_of.as_deref(), nf.preserve_on_behalf_of.unwrap_or(false), &authed, - &authed.email, + &windmill_common::users::username_to_permissioned_as(&authed.username), ) { audit_log( &mut *tx, @@ -1335,6 +1421,7 @@ async fn update_flow( &authed.email, windmill_common::users::username_to_permissioned_as(&authed.username), authed.token_prefix.as_deref(), + authed.username_override.as_deref(), None, None, None, @@ -1410,6 +1497,8 @@ async fn update_flow( new_tx.commit().await?; + reregister_moved_native_triggers(&db, &authed, &w_id, moved_native_triggers); + // Trigger CI tests for items that reference this flow { let db2 = db.clone(); @@ -1512,7 +1601,7 @@ async fn get_flow_by_path( flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, - flow.on_behalf_of_email, + flow.on_behalf_of, flow.labels, folder_labels(flow.workspace_id, flow.path) AS inherited_labels, flow_version.id AS version_id, @@ -1553,7 +1642,7 @@ async fn get_flow_by_path( flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, - flow.on_behalf_of_email, + flow.on_behalf_of, flow.labels, folder_labels(flow.workspace_id, flow.path) AS inherited_labels, flow_version.id AS version_id, @@ -1576,6 +1665,13 @@ async fn get_flow_by_path( tx.commit().await?; + // The primary GET is what the CLI reads before a preserving push, so it must carry the + // derived address: without it the push sends neither half and the identity is cleared. + let mut flow_o = flow_o; + if let Some(fws) = flow_o.as_mut() { + fws.flow.on_behalf_of_email = derived_on_behalf_of_email(&db, &w_id, &fws.flow).await?; + } + // No deployed row + `get_draft`: fall back to the draft table; see scripts.rs. let overlay = overlay_or_draft_only( &db, diff --git a/backend/windmill-api-integration-tests/tests/audit.rs b/backend/windmill-api-integration-tests/tests/audit.rs index 31994d8165..81c3ae70be 100644 --- a/backend/windmill-api-integration-tests/tests/audit.rs +++ b/backend/windmill-api-integration-tests/tests/audit.rs @@ -33,3 +33,105 @@ async fn test_audit_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// A run fired by a labeled token must be findable both by the token that fired it and by the +/// caller who fired it. `push` builds its own audit author from `(user, permissioned_as)` rather +/// than from the `ApiAuthed`, so the label only reaches the row through the explicit end-user +/// argument; and when the runnable declares `on_behalf_of`, the run-as identity takes `username`, +/// so the caller only stays searchable through the `created_by` parameter. +/// +/// EE-only: the OSS `audit_log` writes nothing, and a lesser plan redacts the `parameters` this +/// matches on. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_job_run_is_searchable_by_token_and_by_caller( + db: Pool, +) -> anyhow::Result<()> { + use serde_json::json; + + initialize_tracing().await; + + // The recorded identity makes a run against this take `u/test-user-2` as its + // permissioned_as, so the audit `username` slot goes to it rather than to the caller. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + on_behalf_of, schema, summary, description, lock, extra_perms) + VALUES ('test-workspace', 900101, 'u/test-user-2/onbehalf', 'export function main() {}', + 'deno', 'script', 'test-user-2', 'u/test-user-2', '{}', '', '', '', '{\"g/all\": true}')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = authed(client().post(format!("http://localhost:{port}/api/users/tokens/create"))) + .json(&json!({ "label": "audit-probe" })) + .send() + .await?; + assert_eq!(resp.status(), 201); + let token = resp.text().await?; + let bearer = |b: reqwest::RequestBuilder| b.header("Authorization", format!("Bearer {token}")); + + let resp = bearer(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/preview" + ))) + .json(&json!({ + "content": "export function main() { return 1; }", + "language": "deno", + "args": {} + })) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /jobs/run/preview", + ); + + let resp = bearer(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/p/u/test-user-2/onbehalf" + ))) + .json(&json!({})) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /jobs/run/p/u/test-user-2/onbehalf", + ); + + let search = |q: &str| { + let url = format!("http://localhost:{port}/api/w/test-workspace/audit/list?username={q}"); + async move { + let resp = authed(client().get(url)).send().await.unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await.unwrap() + } + }; + + let by_token = search("label-audit-probe").await; + assert!( + by_token + .iter() + .any(|l| l["operation"] == "jobs.run.preview"), + "the run must be searchable by token label, got {by_token:?}" + ); + assert!( + by_token + .iter() + .any(|l| l["operation"] == "jobs.run.script" && l["username"] == "test-user-2"), + "the on-behalf run must be searchable by token label, got {by_token:?}" + ); + + // `username` is `test-user-2` on that row, so this can only match through `created_by`. + let by_caller = search("test-user").await; + assert!( + by_caller + .iter() + .any(|l| l["operation"] == "jobs.run.script" && l["username"] == "test-user-2"), + "the on-behalf run must stay searchable by the caller who fired it, got {by_caller:?}" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs new file mode 100644 index 0000000000..93c9f5adaf --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs @@ -0,0 +1,118 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// A principal only means something in the workspace whose `usr`/`group_` rows define it, and a +/// fork copies the creator and the groups but not the rest of the membership. Carrying one over +/// blindly would leave a runnable naming somebody who cannot authenticate there; dropping them +/// all would silently hand every configured runnable in the fork to whoever runs it. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_fork_keeps_only_resolvable_on_behalf_of(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base_url = format!("http://localhost:{port}/api"); + + // A superadmin with no `usr` row anywhere: they authenticate from `password` alone, so + // their principal resolves in the fork as much as it did in the parent. + sqlx::query!( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('sa@windmill.dev', 'not-a-real-hash', 'password', true, true, 'Ext', 'ext-sa')" + ) + .execute(&db) + .await?; + + sqlx::query!( + "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of, on_behalf_of_email) + VALUES + ('test-workspace', 'u/test-user/obo_member', 91001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user', 'test@windmill.dev'), + ('test-workspace', 'u/test-user/obo_stranger', 91002, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2', 'test2@windmill.dev'), + ('test-workspace', 'u/test-user/obo_group', 91003, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'g/all', 'group-all@windmill.dev'), + ('test-workspace', 'u/test-user/obo_superadmin', 91004, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/ext-sa', 'sa@windmill.dev'), + ('test-workspace', 'u/test-user/obo_address_only', 91005, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), NULL, 'test2@windmill.dev')" + ) + .execute(&db) + .await?; + + sqlx::query!( + "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, on_behalf_of, on_behalf_of_email) + VALUES ('test-workspace', 'u/test-user/obo_flow', '', '', $1, 'test-user', NOW(), 'u/test-user-2', 'test2@windmill.dev')", + json!({"modules": []}) + ) + .execute(&db) + .await?; + + let resp = reqwest::Client::new() + .post(format!( + "{base_url}/w/test-workspace/workspaces/create_fork" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "id": "wm-fork-obo", "name": "Fork", "color": "#0000ff" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "creating the fork: {}", + resp.text().await? + ); + + let cloned = sqlx::query!( + "SELECT path, on_behalf_of, on_behalf_of_email FROM script WHERE workspace_id = 'wm-fork-obo' ORDER BY path" + ) + .fetch_all(&db) + .await?; + let identity = |path: &str| { + cloned + .iter() + .find(|r| r.path == path) + .unwrap_or_else(|| panic!("{path} was cloned")) + .on_behalf_of + .clone() + }; + + // The creator is the one member the fork always gets, so their principal still resolves. + assert_eq!( + identity("u/test-user/obo_member").as_deref(), + Some("u/test-user") + ); + // Groups are cloned wholesale, so a group principal resolves too. + assert_eq!(identity("u/test-user/obo_group").as_deref(), Some("g/all")); + assert_eq!( + identity("u/test-user/obo_superadmin").as_deref(), + Some("u/ext-sa") + ); + // `test-user-2` is not carried into the fork, so nothing there can run as them — and the + // address has to go with the principal, or a worker that reads only the address still would. + assert_eq!(identity("u/test-user/obo_stranger"), None); + // A row a server predating this release wrote carries the address alone; the clone must not + // mistake it for one it orphaned, because that address is all a later re-derivation has. + assert_eq!( + cloned + .iter() + .find(|r| r.path == "u/test-user/obo_address_only") + .and_then(|r| r.on_behalf_of_email.as_deref()), + Some("test2@windmill.dev"), + ); + + let orphaned = cloned + .iter() + .filter(|r| { + r.on_behalf_of.is_none() + && r.on_behalf_of_email.is_some() + && r.path != "u/test-user/obo_address_only" + }) + .count(); + assert_eq!(orphaned, 0, "a dropped principal leaves no address behind"); + assert_eq!( + sqlx::query_scalar!( + "SELECT on_behalf_of FROM flow WHERE workspace_id = 'wm-fork-obo'" + ) + .fetch_one(&db) + .await?, + None + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index 1d809ae526..17c6521309 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -17,8 +17,8 @@ use windmill_native_triggers::{ decrypt_oauth_data, delete_native_trigger, delete_workspace_integration, get_workspace_integration, google::{parse_stop_channel_params, should_renew_channel}, - require_native_integration_use, store_native_trigger, store_workspace_integration, - NativeTriggerConfig, OAuthConfig, ServiceName, + list_native_triggers, require_native_integration_use, store_native_trigger, + store_workspace_integration, NativeTriggerConfig, OAuthConfig, ServiceName, }; // ============================================================================ @@ -50,6 +50,8 @@ fn test_authed() -> ApiAuthed { folders: vec![], scopes: None, username_override: None, + username_override_is_token_label: false, + is_session_token: false, token_prefix: None, read_only: false, } @@ -569,6 +571,92 @@ async fn test_cleanup_preserves_triggers(db: Pool) -> anyhow::Result<( Ok(()) } +// ============================================================================ +// 5. Runnable rename +// ============================================================================ + +/// A rename has to carry the trigger row onto the new path and report it as moved: listings only +/// return rows whose runnable still exists, so one left behind on the old path disappears from the +/// UI for good, and one not reported keeps a webhook aimed at the old path. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result<()> { + insert_test_script(&db, "f/test/before").await?; + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + &NativeTriggerConfig { + script_path: "f/test/before".to_string(), + is_flow: false, + webhook_token: "abcdefghij1234567890".to_string(), + }, + json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), + None, + ) + .await?; + // An unrelated trigger already sitting on the target path must not be reported as moved. + insert_test_script(&db, "f/test/after").await?; + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-2", + &NativeTriggerConfig { + script_path: "f/test/after".to_string(), + is_flow: false, + webhook_token: "0987654321jihgfedcba".to_string(), + }, + json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), + None, + ) + .await?; + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE script SET path = $1 WHERE workspace_id = 'test-workspace' AND path = $2", + "f/test/after", + "f/test/before", + ) + .execute(&mut *tx) + .await?; + let moved = windmill_common::triggers::update_triggers_script_path( + &mut tx, + "f/test/after", + "f/test/before", + "test-workspace", + false, + ) + .await?; + tx.commit().await?; + + assert_eq!( + moved + .iter() + .map(|t| (t.service_name.as_str(), t.external_id.as_str())) + .collect::>(), + vec![("nextcloud", "ext-1")] + ); + + let triggers = list_native_triggers( + &db, + "test-workspace", + ServiceName::Nextcloud, + None, + None, + Some("f/test/after"), + Some(false), + ) + .await?; + assert_eq!( + triggers.len(), + 2, + "the moved trigger should be listed under the new path" + ); + + Ok(()) +} + // --- parse_stop_channel_params --- #[test] diff --git a/backend/windmill-api-integration-tests/tests/offboarding.rs b/backend/windmill-api-integration-tests/tests/offboarding.rs index be7a94d655..4434e6a2b5 100644 --- a/backend/windmill-api-integration-tests/tests/offboarding.rs +++ b/backend/windmill-api-integration-tests/tests/offboarding.rs @@ -80,6 +80,16 @@ async fn test_offboard_to_user(db: Pool) -> anyhow::Result<()> { .execute(&db) .await?; + // A shared-path script that runs as the departing user. Both halves of its identity have to + // move: a worker predating MIN_VERSION_SUPPORTS_ON_BEHALF_OF_PRINCIPAL reads the address and + // nothing else, so a stale one keeps running it under someone who has just been offboarded. + sqlx::query!( + "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of, on_behalf_of_email) + VALUES ('test-workspace', 'f/shared/obo', 1099, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2', 'test2@windmill.dev')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(ws_url(port, "offboard/test-user-2"))) .json(&json!({ "reassign_to": "u/test-user", @@ -101,6 +111,17 @@ async fn test_offboard_to_user(db: Pool) -> anyhow::Result<()> { assert!(summary["variables_reassigned"].as_i64().unwrap() > 0); assert!(summary["schedules_reassigned"].as_i64().unwrap() > 0); + let obo = sqlx::query!( + "SELECT on_behalf_of, on_behalf_of_email FROM script WHERE path = 'f/shared/obo' AND workspace_id = 'test-workspace'" + ) + .fetch_one(&db) + .await?; + assert_eq!( + (obo.on_behalf_of.as_deref(), obo.on_behalf_of_email.as_deref()), + (Some("u/test-user"), Some("test@windmill.dev")), + "the reassignment moves the whole identity, not just the half this release reads" + ); + // Verify scripts moved let moved = sqlx::query_scalar!( "SELECT COUNT(*) FROM script WHERE path LIKE 'u/test-user/%' AND workspace_id = 'test-workspace' AND NOT archived AND NOT deleted" diff --git a/backend/windmill-api-integration-tests/tests/token_hash.rs b/backend/windmill-api-integration-tests/tests/token_hash.rs index 45d324a3d4..285f15a9c6 100644 --- a/backend/windmill-api-integration-tests/tests/token_hash.rs +++ b/backend/windmill-api-integration-tests/tests/token_hash.rs @@ -343,10 +343,17 @@ async fn test_rotate_webhook_token(db: Pool) -> anyhow::Result<()> { .execute(&db) .await?; - // Rotate the token - let rotated = rotate_webhook_token(&db, &original_hash, ServiceName::Google) - .await? - .expect("rotate must return Some for existing token"); + // Rotate onto a different runnable than the original token was minted for: a rename moves the + // trigger, and rotation has to follow it rather than carry the old scopes forward. + let renamed_scopes = vec!["jobs:run:flows:f/test/renamed".to_string()]; + let rotated = rotate_webhook_token( + &db, + &original_hash, + ServiceName::Google, + renamed_scopes.clone(), + ) + .await? + .expect("rotate must return Some for existing token"); // New token should be different assert_ne!(rotated.new_token, original_token); @@ -355,7 +362,7 @@ async fn test_rotate_webhook_token(db: Pool) -> anyhow::Result<()> { // New token's hash should exist in DB with the per-service label and expiration let new_hash = hash_token(&rotated.new_token); let new_row = sqlx::query!( - "SELECT label, expiration FROM token WHERE token_hash = $1", + "SELECT label, expiration, scopes FROM token WHERE token_hash = $1", new_hash ) .fetch_optional(&db) @@ -373,6 +380,13 @@ async fn test_rotate_webhook_token(db: Pool) -> anyhow::Result<()> { new_row.expiration.is_some(), "rotated Google token must carry an expiration" ); + // Carrying the old token's scopes here is what made every post-rename retry mint a token no + // callback could use, while reporting success. + assert_eq!( + new_row.scopes.as_deref(), + Some(renamed_scopes.as_slice()), + "rotation must scope the new token to the runnable it was rotated for" + ); // Old token should still exist (deletion deferred to caller) let old_exists: bool = sqlx::query_scalar!( @@ -402,7 +416,7 @@ async fn test_rotate_webhook_token(db: Pool) -> anyhow::Result<()> { assert!(!old_gone, "old token must be gone after explicit deletion"); // Rotating a non-existent hash should return None - let result = rotate_webhook_token(&db, "nonexistent_hash", ServiceName::Google).await?; + let result = rotate_webhook_token(&db, "nonexistent_hash", ServiceName::Google, vec![]).await?; assert!( result.is_none(), "rotating a non-existent token must return None" diff --git a/backend/windmill-api-integration-tests/tests/token_label_idor.rs b/backend/windmill-api-integration-tests/tests/token_label_idor.rs index 92dada92c2..ddec1a481a 100644 --- a/backend/windmill-api-integration-tests/tests/token_label_idor.rs +++ b/backend/windmill-api-integration-tests/tests/token_label_idor.rs @@ -179,6 +179,9 @@ async fn test_trigger_token_labels_still_creatable(db: Pool) -> anyhow "http-test-user-2-cd34", "email-test-user-2-ef56", "my-ci-token", + // Minted client-side by the editor (every TypeScript editor load) and the debugger. + "Ephemeral lsp token", + "debugger-token", ] { let resp = create_token_with_label(port, "SECRET_TOKEN_2", label).await; assert_eq!( @@ -190,3 +193,31 @@ async fn test_trigger_token_labels_still_creatable(db: Pool) -> anyhow Ok(()) } + +/// The mirror of the above: reserved namespaces must NOT be mintable. `username_override_from_label` +/// trusts these shapes to name the entity acting, so a forged one would stamp an arbitrary +/// name onto `v2_job.created_by` and the audit `end_user` — on an `on_behalf_of` runnable, +/// which also takes the `username`/`email` columns, that leaves no trace of the real caller. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_reserved_token_labels_not_creatable(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + for label in [ + "ephemeral-webhook-forged", + "ephemeral-script-end-user-svcaccount", + "ephemeral-script", + "session", + "mcp-oauth-forged", + ] { + let resp = create_token_with_label(port, "SECRET_TOKEN_2", label).await; + assert_eq!( + resp.status(), + 400, + "creating a token with reserved label {label:?} must be rejected" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index 7b3457cde1..67b7508d6c 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -660,3 +660,175 @@ async fn test_change_user_email(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// A superadmin acting outside every workspace has no username of their own, so their runnables +/// name them by their address — and an address may contain a `/`, which every reader of a +/// principal splits on. Moving such an account has to leave behind the form that decodes back to +/// them rather than one that reads as a group. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_change_user_email_to_slash_address(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let global_base = format!("http://localhost:{}/api/users", server.addr.port()); + + sqlx::query!( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('ext@windmill.dev', 'not-a-real-hash', 'password', true, true, 'Ext')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of) + VALUES ('test-workspace', 'u/test-user/s', 93001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'ext@windmill.dev')" + ) + .execute(&db) + .await?; + + // The principal follows the address, and a job row carries it in a narrower column than the + // runnable does, so a move that would make it unenqueueable is refused rather than silently + // leaving runnables that look configured and cannot start. + let resp = authed(client().post(format!("{global_base}/change_email/ext@windmill.dev"))) + .json(&json!({ "new_email": "a-very-long-superadmin-address-for-this-test@windmill.dev" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "{body}"); + assert!(body.contains("characters a job can carry"), "{body}"); + + let resp = authed(client().post(format!("{global_base}/change_email/ext@windmill.dev"))) + .json(&json!({ "new_email": "ops/alice@windmill.dev" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "change_email: {}", resp.text().await?); + + assert_eq!( + sqlx::query_scalar!( + "SELECT on_behalf_of FROM script WHERE path = 'u/test-user/s' AND workspace_id = 'test-workspace'" + ) + .fetch_one(&db) + .await? + .as_deref(), + Some("u/ops/alice@windmill.dev"), + "left bare, the new address would come back as group 'alice@windmill.dev'" + ); + + Ok(()) +} + +/// A group's synthetic address (`group-{name}@windmill.dev`) can also be a real user's, and a +/// runnable configured for the *group* carries that address next to `g/{name}`. Moving the +/// colliding user's account must leave it alone: rewriting one half of the pair would leave it +/// naming two different people, which the deploy path rejects. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_change_user_email_leaves_group_identities(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let global_base = format!("http://localhost:{}/api/users", server.addr.port()); + + sqlx::query!("UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + sqlx::query!("UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO group_(workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}')" + ) + .execute(&db) + .await?; + + // An email change never moves a username, so the principal these rows hold stays put. What + // moves is the address beside it — kept for the workers that still read it — and in every + // pair a group-owned identity keeps the group's synthetic address even though a real account + // now holds it: rewriting one half leaves the pair naming two accounts. + sqlx::query!( + "INSERT INTO app(workspace_id, path, summary, policy, versions) + VALUES ('test-workspace', 'u/test-user/g', '', '{\"on_behalf_of\": \"g/ops\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::jsonb, '{}'), + ('test-workspace', 'u/test-user/u', '', '{\"on_behalf_of\": \"u/test-user-2\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::jsonb, '{}')" + ) + .execute(&db) + .await?; + + sqlx::query!( + "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, on_behalf_of, on_behalf_of_email) + VALUES ('test-workspace', 'u/test-user/sg', 95001, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'g/ops', 'group-ops@windmill.dev'), + ('test-workspace', 'u/test-user/su', 95002, 'def main(): pass', '', '', 'python3', 'test-user', NOW(), 'u/test-user-2', 'group-ops@windmill.dev')" + ) + .execute(&db) + .await?; + + sqlx::query!( + "INSERT INTO draft(workspace_id, path, typ, value, email) + VALUES ('test-workspace', 'u/test-user/dg', 'script', '{\"on_behalf_of\": \"g/ops\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::json, 'test@windmill.dev'), + ('test-workspace', 'u/test-user/du', 'script', '{\"on_behalf_of\": \"u/test-user-2\", \"on_behalf_of_email\": \"group-ops@windmill.dev\"}'::json, 'test@windmill.dev')" + ) + .execute(&db) + .await?; + + let resp = authed(client().post(format!("{global_base}/change_email/group-ops@windmill.dev"))) + .json(&json!({ "new_email": "renamed@windmill.dev" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "change_email: {}", resp.text().await?); + + let apps = sqlx::query!( + "SELECT path, policy->>'on_behalf_of_email' AS email FROM app WHERE workspace_id = 'test-workspace' ORDER BY path" + ) + .fetch_all(&db) + .await?; + assert_eq!( + apps.iter() + .map(|r| (r.path.as_str(), r.email.as_deref())) + .collect::>(), + vec![ + ("u/test-user/g", Some("group-ops@windmill.dev")), + ("u/test-user/u", Some("renamed@windmill.dev")), + ], + "the group-owned app keeps the group's address; the user-owned one moves" + ); + + let scripts = sqlx::query!( + "SELECT path, on_behalf_of_email AS email FROM script WHERE workspace_id = 'test-workspace' AND path LIKE 'u/test-user/s%' ORDER BY path" + ) + .fetch_all(&db) + .await?; + assert_eq!( + scripts + .iter() + .map(|r| (r.path.as_str(), r.email.as_deref())) + .collect::>(), + vec![ + ("u/test-user/sg", Some("group-ops@windmill.dev")), + ("u/test-user/su", Some("renamed@windmill.dev")), + ], + "the group-owned script keeps the group's address; the user-owned one moves" + ); + + let drafts = sqlx::query!( + "SELECT path, value->>'on_behalf_of_email' AS email, value->>'on_behalf_of' AS principal FROM draft WHERE workspace_id = 'test-workspace' ORDER BY path" + ) + .fetch_all(&db) + .await?; + assert_eq!( + drafts + .iter() + .map(|r| (r.path.as_str(), r.principal.as_deref(), r.email.as_deref())) + .collect::>(), + vec![ + ( + "u/test-user/dg", + Some("g/ops"), + Some("group-ops@windmill.dev") + ), + ( + "u/test-user/du", + Some("u/test-user-2"), + Some("renamed@windmill.dev") + ), + ], + "a draft's pair moves as a whole or not at all — either half left behind is a 400 on deploy" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 851dcbb4cd..e9eeb88996 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -964,3 +964,46 @@ async fn test_get_imports(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// A dbt project names a warehouse and nothing else, so the workspace setting is +/// the only place the connection is decided. Two things have to hold for that to +/// work at all: the setting round-trips as the MAP the resolver reads (an +/// envelope stored verbatim makes every warehouse name unresolvable), and the +/// route that serves the name to a worker with no database stays job-scoped. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_warehouses(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{base}/workspaces/edit_dbt_warehouses"))) + .json(&json!({ + "dbt_warehouses": { "main": { "resource_path": "u/admin/wh", "target": "prod" } } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed(client().get(format!("{base}/workspaces/get_settings"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let settings = resp.json::().await?; + assert_eq!( + settings["dbt_warehouses"], + json!({ "main": { "resource_path": "u/admin/wh", "target": "prod" } }) + ); + + // A user token is not a job token: the warehouses a workspace configures are + // a running job's business, not a browsable list. + let resp = authed(client().get(format!("{base}/dbt/warehouse/main"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index b4707fca9f..ebee1bcbbf 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -691,6 +691,7 @@ pub async fn run_flow<'c>( bool, Option>, )> { + let on_behalf_of = flow_version_info.on_behalf_of(w_id, &db).await?; let FlowVersionInfo { version, tag, @@ -698,8 +699,6 @@ pub async fn run_flow<'c>( has_preprocessor, has_failure_module, chat_input_enabled, - on_behalf_of_email, - edited_by, early_return, labels, .. @@ -719,10 +718,10 @@ pub async fn run_flow<'c>( Some(authed.clone().into()), PushIsolationLevel::Transaction(tx), ) - } else if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { + } else if let Some(obo) = on_behalf_of.as_ref() { ( - on_behalf_of_email, - username_to_permissioned_as(&edited_by), + &obo.email, + obo.permissioned_as.clone(), None, PushIsolationLevel::IsolatedRoot(db.clone()), ) @@ -752,6 +751,7 @@ pub async fn run_flow<'c>( email, permissioned_as, authed.token_prefix.as_deref(), + authed.username_override.as_deref(), scheduled_for, None, run_query.parent_job, @@ -769,7 +769,7 @@ pub async fn run_flow<'c>( push_authed.as_ref(), false, None, - trigger, + authed.trigger_or_fallback(trigger), run_query.suspended_mode, ) .await?; @@ -966,6 +966,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( email, permissioned_as, authed.token_prefix.as_deref(), + authed.username_override.as_deref(), scheduled_for, None, run_query.parent_job, @@ -988,7 +989,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( push_authed.as_ref(), false, None, - trigger, + authed.trigger_or_fallback(trigger), run_query.suspended_mode, ) .await?; diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index 0d96a66750..6c2d776e30 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -455,6 +455,7 @@ impl From for Job { resolved_at: None, resolution_note: None, resolved_automatically: None, + trigger_kind: None, }, )), "QueuedJob" => Job::QueuedJob(JobExtended::new( @@ -505,6 +506,7 @@ impl From for Job { runnable_settings_handle: uj.runnable_settings_handle, labels: uj.labels, is_retry: uj.is_retry, + trigger_kind: None, }, )), t => panic!("job type {} not valid", t), diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml index ecf0ff0b61..8b34512872 100644 --- a/backend/windmill-api-scripts/Cargo.toml +++ b/backend/windmill-api-scripts/Cargo.toml @@ -10,12 +10,14 @@ path = "src/lib.rs" [features] default = [] -enterprise = ["windmill-common/enterprise"] +enterprise = ["windmill-common/enterprise", "windmill-native-triggers?/enterprise"] private = ["windmill-common/private", "windmill-dep-map/private"] python = ["dep:windmill-parser-py", "dep:windmill-parser-py-asset"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] +native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger"] [dependencies] windmill-common = { workspace = true, default-features = false } +windmill-native-triggers = { workspace = true, optional = true } windmill-object-store.workspace = true windmill-api-auth.workspace = true windmill-queue.workspace = true diff --git a/backend/windmill-api-scripts/src/asset_inference.rs b/backend/windmill-api-scripts/src/asset_inference.rs index 5debb0d271..9c503ae1ae 100644 --- a/backend/windmill-api-scripts/src/asset_inference.rs +++ b/backend/windmill-api-scripts/src/asset_inference.rs @@ -42,6 +42,13 @@ fn parse_assets_for_lang( #[cfg(feature = "python")] ScriptLang::Python3 => windmill_parser_py_asset::parse_assets(content), ScriptLang::Ansible => windmill_parser_yaml::parse_assets(content), + // dbt is the one language whose assets are not a function of the script + // content: they come from `manifest.json`, which needs a clone of the + // project's repo and a dbt invocation. The deploy-time dependency job + // does that parse and writes the `asset` rows itself + // (`dbt_executor::dbt_dep`), so returning None here is what keeps this + // path from clobbering them with an empty list. + ScriptLang::Dbt => return None, _ => return None, }; match parsed { diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index c2f023ce5a..722e74ee55 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -71,6 +71,7 @@ use windmill_common::{ ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptModule, ScriptWithStarred, }, + triggers::MovedNativeTrigger, users::username_to_permissioned_as, utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath}, worker::to_raw_value, @@ -499,6 +500,35 @@ async fn get_top_hub_scripts( Ok::<_, Error>((status_code, headers, response)) } +/// Re-point the webhooks of the native triggers a rename carried onto the new path. +/// +/// Runs after the deploy transaction commits — repointing a webhook is not undoable — and off the +/// request, because it waits on a third-party service that may be slow or gone, and a deploy that +/// already committed must not look like it failed. The rename itself marked these rows +/// `REREGISTRATION_PENDING`, so nothing is lost silently if this never finishes. +fn reregister_moved_native_triggers( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + moved: Vec, +) { + if moved.is_empty() { + return; + } + #[cfg(feature = "native_trigger")] + { + let (db, authed, w_id) = (db.clone(), authed.clone(), w_id.to_string()); + tokio::spawn(async move { + windmill_native_triggers::rename::reregister_triggers_after_rename( + &db, &authed, &w_id, &moved, + ) + .await; + }); + } + #[cfg(not(feature = "native_trigger"))] + let _ = (db, authed, w_id, moved); +} + async fn create_snapshot_script( authed: ApiAuthed, Extension(user_db): Extension, @@ -513,6 +543,7 @@ async fn create_snapshot_script( let mut tx = None; let mut uploaded = false; let mut handle_deployment_metadata = None; + let mut moved_native_triggers = Vec::new(); while let Some(field) = multipart.next_field().await.unwrap() { let name = field.name().unwrap().to_string(); let data = field.bytes().await.unwrap(); @@ -520,7 +551,7 @@ async fn create_snapshot_script( let ns: NewScript = Some(serde_json::from_slice(&data).map_err(to_anyhow)?).unwrap(); let is_tar = ns.codebase.as_ref().is_some_and(|x| x.ends_with(".tar")); let use_esm = ns.codebase.as_ref().is_some_and(|x| x.contains(".esm")); - let (new_hash, ntx, hdm) = create_script_internal( + let (new_hash, ntx, hdm, moved) = create_script_internal( ns, w_id.clone(), authed.clone(), @@ -540,6 +571,7 @@ async fn create_snapshot_script( script_hash = Some(nh); tx = Some(ntx); handle_deployment_metadata = hdm; + moved_native_triggers = moved; } if name == "file" { let hash = script_hash.as_ref().ok_or_else(|| { @@ -570,6 +602,7 @@ async fn create_snapshot_script( } tx.unwrap().commit().await?; + reregister_moved_native_triggers(&db, &authed, &w_id, moved_native_triggers); if let Some(hdm) = handle_deployment_metadata { hdm.handle(&db).await?; } @@ -627,7 +660,8 @@ async fn create_script( let script_path = ns.path.clone(); let email = authed.email.clone(); let username = authed.username.clone(); - let (hash, tx, hdm) = create_script_internal( + let authed_for_triggers = authed.clone(); + let (hash, tx, hdm, moved_native_triggers) = create_script_internal( ns, w_id.clone(), authed, @@ -638,6 +672,7 @@ async fn create_script( ) .await?; tx.commit().await?; + reregister_moved_native_triggers(&db, &authed_for_triggers, &w_id, moved_native_triggers); if let Some(hdm) = hdm { // hdm is Some when no lock generation is needed (script is ready immediately). // Trigger CI tests for any items that reference this script. @@ -708,6 +743,7 @@ impl HandleDeploymentMetadata { async fn is_noop_deploy_against_parent( ns: &NewScript, parent: &Script, + resolved_on_behalf_of: Option<&str>, db: &DB, ) -> Result { if parent.archived || parent.deleted { @@ -750,7 +786,10 @@ async fn is_noop_deploy_against_parent( auto_kind: _, codebase, has_preprocessor, - on_behalf_of_email, + // both halves are folded into `resolved_on_behalf_of` before the comparison below, + // which is the identity that would actually be stored + on_behalf_of_email: _, + on_behalf_of: _, // caller-intent flag (permission preservation), not script state preserve_on_behalf_of: _, assets, @@ -769,7 +808,20 @@ async fn is_noop_deploy_against_parent( if content != &parent.content { return Ok(false); } - if normalize_optional_text(lock.as_deref()) != normalize_optional_text(parent.lock.as_deref()) { + // A dbt lock is DERIVED, never supplied: the request carries none — the CLI + // sends `lock: undefined` and this route discards any anyway — while a + // deployed parent carries what its dependency job wrote. Comparing them makes + // every unchanged push a new version, a fresh dependency job and the sync + // activity `skip_if_noop` exists to prevent. The parent must actually hold + // one, or a deploy whose dependency job failed could never be retried by + // pushing the same project again. + if matches!(language, ScriptLang::Dbt) { + if normalize_optional_text(parent.lock.as_deref()).is_empty() { + return Ok(false); + } + } else if normalize_optional_text(lock.as_deref()) + != normalize_optional_text(parent.lock.as_deref()) + { return Ok(false); } if summary != &parent.summary { @@ -815,10 +867,25 @@ async fn is_noop_deploy_against_parent( { return Ok(false); } - if on_behalf_of_email != &parent.on_behalf_of_email { + if resolved_on_behalf_of != parent.on_behalf_of.as_deref() { return Ok(false); } - if !schema_opt_eq(schema.as_ref(), parent.schema.as_ref()) { + // Both of a dbt script's derived fields are compared as they WOULD BE STORED, + // not as they arrived: the schema comes from the descriptor and the clients + // cannot derive it (`windmill-parser-wasm` has no dbt arm), so they send the + // previous version's or none at all. Comparing what they sent makes every + // unchanged push differ. + let dbt_schema = matches!(language, ScriptLang::Dbt) + .then(|| windmill_parser_yaml::dbt_arg_schema(content).ok()) + .flatten() + .and_then(|v| serde_json::value::to_raw_value(&v).ok()) + .map(|v| Schema(sqlx::types::Json(v))); + let effective_schema = if matches!(language, ScriptLang::Dbt) { + dbt_schema.as_ref() + } else { + schema.as_ref() + }; + if !schema_opt_eq(effective_schema, parent.schema.as_ref()) { return Ok(false); } if !json_serialize_eq(assets, &parent.assets) { @@ -906,6 +973,7 @@ async fn create_script_internal<'c>( ScriptHash, Transaction<'c, Postgres>, Option, + Vec, )> { if authed.is_operator { return Err(Error::NotAuthorized( @@ -965,7 +1033,8 @@ async fn create_script_internal<'c>( // Apply folder default_permissioned_as the first time a script is deployed // at this path. Check inside the transaction to avoid TOCTOU with concurrent deploys. - let explicit_preserve = ns.on_behalf_of_email.is_some() + let explicit_preserve = (ns.on_behalf_of_email.is_some() + || ns.on_behalf_of.is_some()) && ns.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed); if !explicit_preserve && windmill_common::can_preserve_on_behalf_of(&authed) { @@ -978,17 +1047,33 @@ async fn create_script_internal<'c>( .await? .unwrap_or(false); if !path_already_exists { - if let Some(default_email) = - windmill_common::folders::resolve_folder_default_on_behalf_of_email( - &db, &w_id, &ns.path, - ) - .await? + if let Some((default_email, default_permissioned_as)) = + windmill_common::folders::resolve_folder_default_on_behalf_of(&db, &w_id, &ns.path) + .await? { ns.on_behalf_of_email = Some(default_email); + ns.on_behalf_of = Some(default_permissioned_as); ns.preserve_on_behalf_of = Some(true); } } } + let authed_principal = windmill_common::users::username_to_permissioned_as(&authed.username); + // Resolved here rather than at the INSERT so the no-op check below compares the identity + // that would actually be stored: the parent row holds only the principal, while a + // preserving push may name that same principal by address alone. + let resolved_on_behalf_of = windmill_common::resolve_on_behalf_of( + ns.on_behalf_of_email.as_deref(), + ns.on_behalf_of.as_deref(), + ns.preserve_on_behalf_of.unwrap_or(false), + &authed, + &w_id, + &db, + ) + .await?; + // Written beside the principal only while a worker that still reads it may be live. + let legacy_on_behalf_of_email = + windmill_common::legacy_on_behalf_of_email(resolved_on_behalf_of.as_deref(), &w_id, &db) + .await?; if sqlx::query_scalar!( "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", hash.0, @@ -1089,17 +1174,31 @@ async fn create_script_internal<'c>( // sync / promotion callbacks — the whole point is that idempotent // CLI pushes must not produce phantom commits on the downstream // git repository. - if skip_if_noop && is_noop_deploy_against_parent(&ns, &ps, &db).await? { + if skip_if_noop + && is_noop_deploy_against_parent( + &ns, + &ps, + resolved_on_behalf_of.as_deref(), + &db, + ) + .await? + { tracing::info!( workspace_id = %w_id, path = %ns.path, parent_hash = %p_hash.0, "Skipping no-op script deploy (identical to parent)" ); - return Ok((p_hash.clone(), tx, None)); + return Ok((p_hash.clone(), tx, None, Vec::new())); } if ps.path != ns.path { + // A rename writes the source as much as the destination, and only the destination + // is scope-checked above. `require_owner_of_path` answers whether the *user* owns + // the source, never what their token is scoped to — so without this a path-scoped + // token could move a script it has no say over, taking its native triggers along + // and re-registering them under that token's identity. + check_scopes(&authed, || format!("scripts:write:{}", ps.path))?; require_owner_of_path(&authed, &ps.path)?; } @@ -1143,6 +1242,25 @@ async fn create_script_internal<'c>( .as_ref() .map(|v| v.perms.clone()) .unwrap_or(json!({})); + // A dbt lock names a manifest digest and engine versions that only a + // dependency job can determine, and that job is also what publishes the + // script's manifest graph. Honouring a supplied one would skip it, leaving + // the script with no graph — including on the UI paths that round-trip an + // existing lock, like rename and unarchive. + if matches!(ns.language, ScriptLang::Dbt) { + ns.lock = None; + // A codebase is a bundle of JS/TS sources; a dbt script's project is + // its modules. Accepting one takes the branch below that stands in for + // lock generation, which would suppress the very job that parses the + // project and publishes the graph. + if ns.codebase.is_some() { + return Err(Error::BadRequest( + "a dbt script has no codebase: its project is its modules, the \ + `` in + // any value (the app path comes from raw_app.yaml) would end the block. + const config = JSON.stringify({ + appPath: opts.appPath, + workspace: opts.workspace, + playerBaseUrl: opts.playerBaseUrl ?? null, + savePath: RECORDER_SAVE_PATH, + }).replace(/ + + + + + Windmill App Dev Recording + + + +
+ + Not recording + + + Passwords are masked. Mark sensitive elements with data-wm-no-record +
+ + + + + +`; +} diff --git a/cli/src/commands/app/devRecorderBundle.gen.ts b/cli/src/commands/app/devRecorderBundle.gen.ts new file mode 100644 index 0000000000..0e7f004b4b --- /dev/null +++ b/cli/src/commands/app/devRecorderBundle.gen.ts @@ -0,0 +1,15 @@ +// Generated by cli/generate-dev-recorder.ts. Do not edit. +// Run `bun run gen:dev-recorder` from cli/ to rebuild it from +// frontend/src/lib/components/recording/. + +/** Repo-relative sources bundled below. */ +export const DEV_RECORDER_SOURCES = [ + "frontend/src/lib/components/recording/rawAppRecording.svelte.ts", + "frontend/src/lib/components/recording/rawAppSnapshot.ts" +]; + +/** SHA-256 of those sources, as of this build. */ +export const DEV_RECORDER_SOURCE_HASH = "c93b8b23455528be0a03da303fb573536bd3f0b8386e6f5f1078a6b988f59082"; + +/** IIFE exposing `createRawAppRecording` on `window.__wmillRecorder`. */ +export const DEV_RECORDER_BUNDLE = "var __wmillRecorder=(()=>{var ne=Object.defineProperty;var Fe=Object.getOwnPropertyDescriptor;var Xe=Object.getOwnPropertyNames;var Be=Object.prototype.hasOwnProperty;var je=(t,o)=>{for(var n in o)ne(t,n,{get:o[n],enumerable:!0})},Ke=(t,o,n,s)=>{if(o&&typeof o==\"object\"||typeof o==\"function\")for(let l of Xe(o))!Be.call(t,l)&&l!==n&&ne(t,l,{get:()=>o[l],enumerable:!(s=Fe(o,l))||s.enumerable});return t};var We=t=>Ke(ne({},\"__esModule\",{value:!0}),t);var mt={};je(mt,{createRawAppRecording:()=>Ie});var G=\"data-wm-rec-target\",D=\"data-wm-no-record\";function Ye(t,o){let n=(u,c,f)=>{let d=f.trim();if(!d||/^(data:|blob:|about:|https?:|\\/\\/|#)/i.test(d))return u;try{return`url(${c}${new URL(d,o).href}${c})`}catch{return u}},s=\"\",l=0;for(;ln?String.fromCodePoint(parseInt(n,16)):s)}function Qe(t){let o=new Set,n=new Set;for(let s of Array.from(t.querySelectorAll(\"style\"))){if(_(s))continue;let l=s.textContent??\"\";for(let u of l.matchAll(Ge))o.add(Ee(u[1]));for(let u of l.matchAll(Je))n.add(Ee(u[1]))}return{classes:o,ids:n}}function Ze(t,o){let n=Qe(o),s=[...o.hasAttribute(D)?[o]:[],...Array.from(o.querySelectorAll(`[${D}]`))];for(let l of s){l.replaceChildren(t.createTextNode(\"\\u2022\\u2022\\u2022\")),l.setAttribute(D,\"\");for(let u of Array.from(l.attributes)){if(u.name===D)continue;let c=u.localName.toLowerCase();if(!ze.has(c))l.removeAttributeNode(u);else if(c===\"class\"){let f=u.value.split(/\\s+/).filter(d=>d&&n.classes.has(d));f.length?l.setAttribute(\"class\",f.join(\" \")):l.removeAttributeNode(u)}else c===\"id\"&&!n.ids.has(u.value)&&l.removeAttributeNode(u)}}}var et=4e6,tt=8e6;function nt(t,o){let n=t.querySelectorAll(\"canvas\"),s=o.querySelectorAll(\"canvas\");if(n.length!==s.length)return;let l=tt;for(let u=0;uet||f>l)continue;l-=f;let d;try{d=c.toDataURL(\"image/webp\",.85)}catch{continue}if(!d.startsWith(\"data:image/\"))continue;let m=c.getBoundingClientRect();if(!m.width||!m.height)continue;let y=t.defaultView?.getComputedStyle(c).display,v=!y||y===\"inline\"?\"inline-block\":y,F=s[u],X=F.getAttribute(\"style\");F.setAttribute(\"style\",`${X?X+\";\":\"\"}display:${v};box-sizing:border-box;width:${m.width}px;height:${m.height}px;background-image:url(\"${d}\");background-size:100% 100%;background-repeat:no-repeat`)}}function rt(t,o){let n=t.querySelectorAll(\"select\"),s=o.querySelectorAll(\"select\");if(n.length===s.length)for(let l=0;l_(f)))continue;let c=t.createElement(\"option\");c.setAttribute(\"selected\",\"\"),c.textContent=\"\\u2022\\u2022\\u2022\",s[l].replaceChildren(c)}}function it(t,o){let n=\"input, textarea, select\",s=t.querySelectorAll(n),l=o.querySelectorAll(n);if(s.length===l.length)for(let u=0;u{let n=o.styleSheet;if(!n)return o.cssText;try{let s=be(n.cssRules),l=n.media?.mediaText;return l?`@media ${l} {\n${s}\n}`:s}catch{return o.cssText}}).join(`\n`)}function ot(t,o){let n=be(o);t.href&&(n=Ye(n,t.href));let s=t.media?.mediaText;return s&&(n=`@media ${s} {\n${n}\n}`),n}function st(t,o,n){for(let s of Array.from(t.styleSheets)){let l=s.ownerNode;if(!C(l))continue;if(s.disabled){let m=re(o,l),y=m?ie(n,m):void 0;y&&(y.setAttribute(\"media\",\"not all\"),y.tagName===\"STYLE\"&&(y.textContent=\"\"));continue}if(_(l))continue;let u;try{let m=s.cssRules;if(!m)continue;u=m}catch{continue}let c=re(o,l);if(!c)continue;let f=ie(n,c);if(!f)continue;let d=ot(s,u);if(l.tagName===\"LINK\"){let m=t.createElement(\"style\");m.textContent=d,f.replaceWith(m)}else l.tagName===\"STYLE\"&&(f.textContent=d)}}function Se(t,o={}){let n=t.documentElement,s=n.cloneNode(!0);if(it(t,s),st(t,n,s),nt(t,s),rt(t,s),Ze(t,s),o.target){let d=re(n,o.target);(d?ie(s,d):void 0)?.setAttribute(G,\"\")}s.querySelectorAll(\"template, noscript\").forEach(d=>d.remove()),s.querySelectorAll(\"script\").forEach(d=>d.remove()),s.querySelectorAll('meta[http-equiv=\"refresh\" i]').forEach(d=>d.remove()),s.querySelectorAll(\"*\").forEach(d=>{for(let m of Array.from(d.attributes))m.name.toLowerCase().startsWith(\"on\")&&d.removeAttribute(m.name)});let l=t.defaultView,u=Math.round(l?.scrollY??t.documentElement.scrollTop??0),c=Math.round(l?.scrollX??t.documentElement.scrollLeft??0);if(u>0||c>0){let d=t.createElement(\"style\");d.textContent=`html { margin-top: -${u}px !important; margin-left: -${c}px !important; }`,s.querySelector(\"head\")?.appendChild(d)}let f=s.querySelector(\"head\");if(o.baseHref&&f&&!f.querySelector(\"base\")){let d=t.createElement(\"base\");d.setAttribute(\"href\",o.baseHref),f.prepend(d)}return`${s.outerHTML}`}var gt=`[${G}] {\n\toutline: 3px solid #ef4444 !important;\n\toutline-offset: 2px !important;\n\tbox-shadow: 0 0 0 6px rgba(239, 68, 68, 0.25) !important;\n}`;function oe(t){if(!t||_(t))return\"\";let o=t;return t.querySelector(`[${D}]`)&&(o=t.cloneNode(!0),o.querySelectorAll(`[${D}]`).forEach(n=>n.remove())),(o.textContent??\"\").replace(/\\s+/g,\" \").trim()}function ye(t,o=40){let n=oe(t);return n.length>o?`${n.slice(0,o)}\\u2026`:n}function we(t){let o=t.tagName.toLowerCase(),n=(t.getAttribute(\"type\")??\"text\").toLowerCase(),s=o===\"input\"?`input[${n}]`:o,l=t.labels?.[0],u=t.getAttribute(\"aria-label\")||(l&&!_(l)?ye(l):\"\")||(o===\"input\"&&[\"button\",\"submit\",\"reset\"].includes(n)?t.getAttribute(\"value\"):\"\")||t.getAttribute(\"placeholder\")||t.getAttribute(\"title\")||ye(t)||t.getAttribute(\"name\")||t.getAttribute(\"id\")||\"\";return u?`${s} \"${u}\"`:s}function Re(t){let o=[],n=t,s=0;for(;n&&s<5;){let l=n.tagName.toLowerCase();if(n.id){o.unshift(`#${n.id}`);break}let u=typeof n.className==\"string\"?n.className.trim().split(/\\s+/).filter(Boolean)[0]:void 0,c=n.parentElement,f=u?`${l}.${u}`:l;if(c){let d=Array.from(c.children).filter(m=>m.tagName===n.tagName);d.length>1&&(f+=`:nth-of-type(${d.indexOf(n)+1})`)}o.unshift(f),n=c,s++}return o.join(\" > \")}function Le(t,o,n){switch(t){case\"click\":return`Clicked ${o}`;case\"fill\":return`Filled ${o} with \"${n??\"\"}\"`;case\"select\":return`Selected \"${n??\"\"}\" in ${o}`;case\"toggle\":return n?`${n===\"checked\"?\"Checked\":\"Unchecked\"} ${o}`:`Toggled ${o}`;case\"submit\":return`Submitted ${o}`;case\"key\":return`Pressed ${n??\"key\"} in ${o}`;case\"navigate\":return n?`Navigated to ${n}`:\"Reloaded the app\"}}var V=400,ct=3e3,ve=6e4,Ce=800,_e=new Set([\"button\",\"submit\",\"reset\",\"image\"]),ut=new Set([\"range\",\"color\",\"date\",\"time\",\"datetime-local\",\"month\",\"week\"]),dt=new Set([\"\",\"text\",\"search\",\"url\",\"tel\",\"email\",\"password\",\"number\"]),J=200,ft=250,xe=500;function Ie(){let t=!1,o=0,n=0,s=\"\",l,u,c=[],f=[],d=new Map,m=0,y=!1,v=!1,F={width:0,height:0},X=\"\",z=[],A,M,O=0,L,b,h,q=new Set,Q,N=0,B,Z=!1;function ke(e){return new Promise(a=>{let i=()=>{if(N===0||Date.now()-e>=ve||!P()){a();return}setTimeout(i,V)};i()})}let Me=e=>new Promise(a=>setTimeout(a,e));function P(){try{return u?.contentDocument??void 0}catch{return}}function $(e){if(e===void 0)return;let a=d.get(e);if(a!==void 0)return a;if(m+e.length>41943040){y=!0,v=!0;return}let i=f.length;return f.push(e),d.set(e,i),m+=e.length,i}function S(e){if(v)return;let a=P();if(a)try{return Se(a,{target:e,baseHref:X})}catch(i){console.warn(\"raw app recorder: snapshot failed\",i);return}}function Ne(e){return e.replace(` ${G}=\"\"`,\"\")}function j(){h&&(h.observer.disconnect(),clearTimeout(h.timer),clearTimeout(h.cap),h=void 0)}function se(e){j();let a=P();if(!a)return;let i=()=>{if(N>0&&h&&Date.now()-h.startedAt{h&&(clearTimeout(h.timer),h.timer=setTimeout(i,V))});r.observe(a,{subtree:!0,childList:!0,attributes:!0,characterData:!0}),h={step:e,observer:r,startedAt:Date.now(),timer:setTimeout(i,V),cap:setTimeout(i,ct)}}function ae(e){if(!h)return;let a=h.step;j();let i=e!==void 0&&(L?.html===e||b?.html===e);a.after=$(i?Ne(e):S())}function x(e,a,i,r,w=!1){if(!t)return;let E=Date.now()-n,p=c[c.length-1],R=!!p&&!!a&&ue(M,a)&&p.kind===e&&(w||Ue(a)&&E-O=500&&!R){y=!0,v=!0;return}let T=!!a&&_(a),H=le(a?T?Ae(a):we(a):\"the app\")??\"the app\",W=r&&r.length>J?`${r.slice(0,J)}\\u2026`:r,k=!T||!W?W:e===\"toggle\"?void 0:Y(W),ge=Le(e,H,k);if(R&&p){p.value=k,p.label=ge,O=E,se(p);return}let he={t:E,kind:e,label:ge,target:H,selector:a&&!T?le(Re(a)):void 0,value:k,before:$(i??(e===\"key\"?S(a):void 0))};c.push(he),i!==void 0&&L?.html===i&&(L=void 0),i!==void 0&&b?.html===i&&(b=void 0),M=a,O=E,o=c.length,se(he)}function Pe(e){let i=e.closest(\"label\")?.control;return!i||e===i||i.contains(e)?!1:!e.closest(\"a, button, input, select, textarea\")}function $e(e){return e.ctrlKey||e.metaKey||e.altKey?!1:e.key.length===1||[\" \",\"Enter\",\"ArrowUp\",\"ArrowDown\",\"ArrowLeft\",\"ArrowRight\",\"Home\",\"End\"].includes(e.key)}function ee(e){if(g(e,\"SELECT\"))return!0;if(!g(e,\"INPUT\"))return!1;let a=e.type;return!K(e)&&!_e.has(a)}function He(e){return g(e,\"BUTTON\")?(e.type||\"submit\")===\"submit\":g(e,\"INPUT\")&&[\"submit\",\"image\"].includes(e.type)}function De(e){let a=c[c.length-1];return a?.kind===\"key\"&&a.value===\"Enter\"&&!!e&&!!M&&e.contains(M)&&Date.now()-n-OJ?`${e.slice(0,J)}\\u2026`:e}function K(e){return g(e,\"TEXTAREA\")||e.isContentEditable?!0:g(e,\"INPUT\")&&dt.has(e.type)}function ce(e){let a=g(e,\"INPUT\")||g(e,\"TEXTAREA\")?e.value:oe(e);return g(e,\"INPUT\")&&e.type===\"password\"||_(e)?Y(a):a}function U(e){let a=L?.el;if(!a)return;if(a===e||a.contains(e)||e.contains(a))return L?.html;let i=e.labels;if(i&&Array.from(i).some(r=>r===a||r.contains(a)))return L?.html}function ue(e,a){if(!e||!a)return!1;if(e===a)return!0;let i=e,r=a;return i.type===\"radio\"&&r.type===\"radio\"&&!!i.name&&i.name===r.name&&i.form===r.form}function te(e){if(b)return ue(b.el,e)?b.html:void 0}function I(){if(!A)return;let{el:e,before:a}=A;clearTimeout(A.timer),A=void 0,U(e)!==void 0&&(L=void 0),b?.el===e&&(b=void 0),x(\"fill\",e,a,ce(e))}function de(e){let a=(i,r)=>{e.addEventListener(i,r,!0),z.push(()=>e.removeEventListener(i,r,!0))};a(\"pointerdown\",i=>{let r=C(i.target)?i.target:void 0;r&&(L={el:r,html:S(r)})}),a(\"click\",i=>{let r=C(i.target)?i.target:void 0;r&&(A&&A.el!==r&&I(),!(K(r)||ee(r)||g(r,\"OPTION\"))&&(i.detail===0&&He(r)&&De(r.closest(\"form\"))||Pe(r)||x(\"click\",r,U(r)??S(r))))}),a(\"focusin\",i=>{let r=C(i.target)?i.target:void 0;L&&(!r||U(r)===void 0)&&(L=void 0)}),a(\"beforeinput\",i=>{let r=C(i.target)?i.target:void 0;!r||!K(r)||A?.el===r||U(r)===void 0&&(b={el:r,html:S(r),repeat:!1})}),a(\"input\",i=>{let r=C(i.target)?i.target:void 0;if(!(!r||!K(r)))if(A&&A.el!==r&&I(),A)clearTimeout(A.timer),A.timer=setTimeout(I,Ce);else{let w=U(r),E=te(r),p=w??E??S(r);ae(p),A={el:r,before:p,timer:setTimeout(I,Ce)}}}),a(\"change\",i=>{let r=C(i.target)?i.target:void 0;if(!r)return;if(K(r)){I();return}A&&I();let w=U(r),E=te(r),p=w??E,R=w===void 0&&E!==void 0&&!!b?.repeat;if(g(r,\"SELECT\")){let T=Array.from(r.selectedOptions),H=T.map(k=>k.label||k.value).join(\", \"),W=T.some(k=>_(k));x(\"select\",r,p,W?Y(H):H,R)}else if(g(r,\"INPUT\")){let T=r;[\"checkbox\",\"radio\"].includes(T.type)?x(\"toggle\",r,p,T.checked?\"checked\":\"unchecked\",R):T.type===\"file\"?x(\"fill\",r,p,Array.from(T.files??[]).map(H=>H.name).join(\", \")):x(\"fill\",r,p,ce(r),R)}}),a(\"submit\",i=>{let r=C(i.target)?i.target:void 0;I();let w=c[c.length-1];(w?.kind===\"click\"||w?.kind===\"key\"&&w.value===\"Enter\")&&M&&r&&r.contains(M)&&Date.now()-n-O{let r=C(i.target)?i.target:void 0;r&&ee(r)&&$e(i)&&(i.repeat&&b&&te(r)!==void 0?b.repeat=!0:b={el:r,html:S(r),repeat:i.repeat}),!(i.key!==\"Enter\"&&i.key!==\"Escape\")&&(i.key===\"Enter\"&&r&&(ee(r)||qe(r)||Oe(r))||(I(),x(\"key\",r,i.repeat?void 0:S(r),i.key,i.repeat)))})}function fe(){z.forEach(e=>e()),z=[]}function me(e){let a=e.contentWindow,i=E=>{let p=E.data;if(!p||typeof p!=\"object\"||E.source!==window)return;let{type:R,reqId:T}=p;typeof R!=\"string\"||!R.endsWith(\"Res\")||(q.delete(T),N=q.size)},r=()=>{let E=P();!E||E===Q||(a?.addEventListener(\"message\",i),Q=E)},w=E=>{let p=E.data;if(!p||typeof p!=\"object\"||E.source!==a)return;let{type:R,reqId:T}=p;typeof R!=\"string\"||R.endsWith(\"Res\")||T===void 0||(r(),q.add(T),N=q.size)};return window.addEventListener(\"message\",w),r(),()=>{window.removeEventListener(\"message\",w),a?.removeEventListener(\"message\",i),Q=void 0}}function pe(){fe();let e=h?.step;j(),A&&clearTimeout(A.timer),A=void 0,L=void 0,b=void 0;let a=P();if(!a)return;de(a),u&&(B?.(),B=me(u));let i=S();if(e&&(e.after=$(i)),f.length===0){$(i);return}x(\"navigate\",void 0,i,a.location?.hash||void 0)}return{get active(){return t},get stepCount(){return o},get stopping(){return Z},start(e,a){u=e;let i=P();return i?.documentElement?(t=!0,n=Date.now(),s=a.appPath,l=a.workspace,c=[],M=void 0,O=0,o=0,f=[],d=new Map,m=0,y=!1,v=!1,X=typeof window<\"u\"?window.location.origin:\"\",F={width:e.clientWidth||i.documentElement.clientWidth,height:e.clientHeight||i.documentElement.clientHeight},i.readyState===\"complete\"&&i.location?.href!==\"about:blank\"&&$(S()),de(i),e.addEventListener(\"load\",pe),q.clear(),N=0,B=me(e),!0):(u=void 0,!1)},async stop(){if(I(),fe(),t=!1,h&&N>0){Z=!0;let a=h.startedAt;await ke(a),P()&&await Me(V),Z=!1}if(h){let a=h.step;j(),a.after=$(S())}B?.(),B=void 0,q.clear(),N=0,u?.removeEventListener(\"load\",pe),L=void 0,b=void 0,u=void 0;let e={version:1,type:\"app\",recorded_at:new Date().toISOString(),app_path:s,workspace:l,total_duration_ms:Date.now()-n,viewport:F,frames:f,steps:c,truncated:y||void 0};return c=[],f=[],d=new Map,m=0,e},download(e){let a=new Blob([JSON.stringify(e)],{type:\"application/json\"}),i=URL.createObjectURL(a),r=document.createElement(\"a\");r.href=i,r.download=`app-recording-${(e.app_path||\"untitled\").replace(/\\//g,\"-\")}-${Date.now()}.json`,r.click(),URL.revokeObjectURL(i)}}}return We(mt);})();\n"; diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index fcfc68915e..9f83d0322a 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -1,5 +1,6 @@ import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; @@ -16,7 +17,7 @@ import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { replaceInlineScripts, repopulateFields } from "./app.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; -import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; +import { APP_BACKEND_FOLDER, RECORDINGS_FOLDER } from "./app_metadata.ts"; import { writeIfChanged } from "../../utils/utils.ts"; import { yamlOptions } from "../sync/sync.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; @@ -143,7 +144,8 @@ function getRunnableIdFromCodeFile(fileName: string): string | undefined { * Returns an empty object if the backend folder doesn't exist. * * @param backendPath - Path to the backend folder - * @param defaultTs - Default TypeScript runtime ("bun" or "deno") + * @param defaultTs - TypeScript runtime a bare `.ts` denotes. Must match what + * newRawAppPathAssigner used to write the file, or the round-trip relabels it. */ export async function loadRunnablesFromBackend( backendPath: string, @@ -317,6 +319,12 @@ async function collectAppFiles( ) { continue; } + // Session recordings, which the dev server only ever writes at the app + // root. Matched there alone, so an app of its own with a `recordings/` + // component folder still ships it. + if (basePath === "/" && entry.name === RECORDINGS_FOLDER) { + continue; + } await readDirRecursive(fullPath + SEP, relativePath + "/"); } else if (entry.isFile()) { // Skip generated/metadata files that shouldn't be part of the app @@ -344,6 +352,7 @@ export async function pushRawApp( remotePath: string, localPath: string, message?: string, + defaultTs: "bun" | "deno" = "bun", ): Promise { if (alreadySynced.includes(localPath)) { return; @@ -377,7 +386,10 @@ export async function pushRawApp( // Load runnables from separate YAML files in the backend folder // Falls back to reading from raw_app.yaml if no separate files exist (backward compat) const backendPath = path.join(localPath, APP_BACKEND_FOLDER); - const runnablesFromBackend = await loadRunnablesFromBackend(backendPath); + const runnablesFromBackend = await loadRunnablesFromBackend( + backendPath, + defaultTs, + ); let runnables: Record; if (Object.keys(runnablesFromBackend).length > 0) { @@ -539,7 +551,14 @@ async function pushRawAppCommand( } const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const merged = await mergeConfigWithConfigFile(opts); - await pushRawApp(workspace.workspaceId, remotePath, filePath); + await pushRawApp( + workspace.workspaceId, + remotePath, + filePath, + undefined, + merged.defaultTs, + ); log.info(colors.bold.underline.green("Raw app pushed")); } diff --git a/cli/src/commands/app/wmillTsDev.ts b/cli/src/commands/app/wmillTsDev.ts index cd77652f53..7cb2ea328b 100644 --- a/cli/src/commands/app/wmillTsDev.ts +++ b/cli/src/commands/app/wmillTsDev.ts @@ -68,11 +68,45 @@ function initWebSocket() { initWebSocket() +/** A runnable call leaves this page over the WebSocket without touching the DOM, + * so the session recorder of \`wmill app dev --recording\` (which frames the app) + * has nothing else to tell it a step is still waiting on the backend. Announcing + * the request and its answer to the shell mirrors what the deployed runner posts + * across the same boundary. */ +const framed = typeof window !== 'undefined' && window.parent !== window + +function notifyRecorder(type: string, reqId: string) { + if (framed) window.parent.postMessage({ type, reqId }, window.location.origin) +} + +// A reload takes the previous context and its WebSocket with it, so whatever it +// had in flight can never answer. Announcing a fresh module is how the shell +// learns those calls are dead: a message posted from the unloading document +// would be dropped with the realm that sent it, and this runs before any app +// code can issue a call of its own. +if (framed) { + window.parent.postMessage({ type: 'wmillDevReady' }, window.location.origin) +} + +function tracked(type: string, reqId: string, resolve: (v: any) => void, reject: (e: any) => void) { + notifyRecorder(type, reqId) + let settled = false + const done = () => { + if (settled) return + settled = true + notifyRecorder(type + 'Res', reqId) + } + return { + resolve: (v: any) => { done(); resolve(v) }, + reject: (e: any) => { done(); reject(e) } + } +} + async function doRequest(type: string, o: object) { await wsReady return new Promise((resolve, reject) => { const reqId = Math.random().toString(36) - reqs[reqId] = { resolve, reject } + reqs[reqId] = tracked(type, reqId, resolve, reject) ws?.send(JSON.stringify({ ...o, type, reqId })) }) } @@ -119,7 +153,7 @@ export function streamJob( return new Promise(async (resolve, reject) => { await wsReady const reqId = Math.random().toString(36) - reqs[reqId] = { resolve, reject, onUpdate } + reqs[reqId] = { ...tracked('streamJob', reqId, resolve, reject), onUpdate } ws?.send(JSON.stringify({ jobId, type: 'streamJob', reqId })) }) } diff --git a/cli/src/commands/dev/dev.ts b/cli/src/commands/dev/dev.ts index f588f5841d..00a6f72f0e 100644 --- a/cli/src/commands/dev/dev.ts +++ b/cli/src/commands/dev/dev.ts @@ -20,7 +20,19 @@ import { SyncOptions, mergeConfigWithConfigFile, } from "../../core/conf.ts"; -import { exts, removeExtensionToPath } from "../script/script.ts"; +import { + exts, + hasScriptExt, + readModulesFromDisk, + removeExtensionToPath, +} from "../script/script.ts"; +import type { ScriptModule } from "../../../gen/types.gen.ts"; +import { + DBT_DESCRIPTOR_NAME, + DBT_MODULE_SUFFIX, + getScriptBasePathFromModulePath, + isDbtModulePath, +} from "../../utils/resource_folders.ts"; import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { OpenFlow } from "../../../gen/types.gen.ts"; import { FlowFile } from "../flow/flow.ts"; @@ -108,7 +120,7 @@ function findFlowFolderPrefix(cpath: string): string | undefined { return undefined; } -async function listWorkspacePaths(): Promise { +export async function listWorkspacePaths(): Promise { // Walk first, capturing each item's metadata file path. Then read summaries in // parallel — one tree pass plus N file reads is faster than a serialized walk. const items: (WmPathItem & { _metaPath?: string })[] = []; @@ -136,6 +148,19 @@ async function listWorkspacePaths(): Promise { items.push({ path: stripFolderSuffix(childRel, APP_SUFFIXES), kind: "raw_app" }); continue; } + // A dbt script IS the project directory: its descriptor is optional, so + // there may be no file here to recognize it by. Not descended into + // either — the project's own `.sql` models would otherwise each be + // listed as a script of their own. + if (entry.name.endsWith(DBT_MODULE_SUFFIX)) { + const base = childRel.slice(0, -DBT_MODULE_SUFFIX.length); + items.push({ + path: base, + kind: "script", + _metaPath: childAbs.slice(0, -DBT_MODULE_SUFFIX.length) + ".script.yaml", + }); + continue; + } await walk(childAbs, childRel); } else if (entry.isFile()) { const matchedExt = exts.find((ext) => entry.name.endsWith(ext)); @@ -282,12 +307,40 @@ export async function dev(opts: GlobalOptions & SyncOptions & DevOpts) { const flowMetadataFile = getMetadataFileName("flow", "yaml"); async function loadPaths(pathsToLoad: string[]) { - const paths = pathsToLoad.filter((p) => - exts.some( - (ext) => p.endsWith(ext) - || p.endsWith(".flow/" + flowMetadataFile) - || p.endsWith("__flow/" + flowMetadataFile) - ) + // A change ANYWHERE inside a dbt project is a change to that script: the + // bundle is the project, so the whole thing is re-read and rebroadcast. + // Treating the file as a script of its own would drop the edit — a bare + // `.sql` has no language to infer, and a `.yml` or `.csv` is filtered out + // below — leaving the browser previewing the snapshot taken at startup. + const rest: string[] = []; + const dbtProjects = new Set(); + for (const raw of pathsToLoad) { + const rel = (await realpath(raw).catch(() => raw)) + .replace(base + SEP, "") + .replaceAll("\\", "/"); + const wmPath = isDbtModulePath(rel) + ? getScriptBasePathFromModulePath(rel) + : undefined; + // Every project in the batch, and each only once: a save-all or a + // `git checkout` touches many files at once, and returning on the first + // would drop both the other projects and whatever else changed with them. + if (wmPath) dbtProjects.add(wmPath); + else rest.push(raw); + } + for (const wmPath of dbtProjects) { + const edit = await loadWmPath(wmPath); + if (edit) { + log.info("Updated " + wmPath + " (dbt project)"); + broadcastChanges(edit); + } + } + if (rest.length === 0) return; + pathsToLoad = rest; + const paths = pathsToLoad.filter( + (p) => + hasScriptExt(p) || + p.endsWith(".flow/" + flowMetadataFile) || + p.endsWith("__flow/" + flowMetadataFile) ); if (paths.length == 0) { return; @@ -380,6 +433,10 @@ export async function dev(opts: GlobalOptions & SyncOptions & DevOpts) { tag?: string; lock?: string; temp_script_refs?: Record; + /** The bundle the dev page forwards to a preview run. A dbt project cannot + * run without it: the worker looks for `dbt_project.yml` in the bundle and + * refuses the job when it is not there. */ + modules?: Record; }; type LastEditFlow = { @@ -441,8 +498,23 @@ export async function dev(opts: GlobalOptions & SyncOptions & DevOpts) { for (const ext of exts) { const filePath = wmPath + ext; try { - await access(filePath); - const content = await readTextFile(filePath); + // A dbt project's descriptor is optional, so what says "this is a dbt + // script" is the project beside it. Requiring the descriptor to exist + // would list such a project in the picker and then refuse to load it. + const isAbsentDbtDescriptor = + ext === "__dbt/" + DBT_DESCRIPTOR_NAME && + !(await access(filePath).then( + () => true, + () => false + )) && + (await access(wmPath + DBT_MODULE_SUFFIX + "/dbt_project.yml").then( + () => true, + () => false + )); + if (!isAbsentDbtDescriptor) { + await access(filePath); + } + const content = isAbsentDbtDescriptor ? "" : await readTextFile(filePath); const lang = inferContentTypeFromFilePath(filePath, opts.defaultTs); const typed = (await parseMetadataFile(removeExtensionToPath(filePath), undefined))?.payload; const edit: LastEditScript = { @@ -453,6 +525,18 @@ export async function dev(opts: GlobalOptions & SyncOptions & DevOpts) { tag: typed?.tag, lock: typed?.lock, temp_script_refs: tempScriptRefs, + // Read VERBATIM for dbt, the way push does: the project's `.sql` and + // `.yml` files are dbt's, and inferring a language for each would + // drop the ones that are not Windmill scripts. + modules: + lang === "dbt" + ? await readModulesFromDisk( + wmPath + DBT_MODULE_SUFFIX, + opts.defaultTs, + true, + true + ) + : undefined, }; currentLastEdit = edit; return edit; diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 17a5e1febb..97da2f6699 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -232,11 +232,19 @@ export async function pushFlow( const hasOnBehalfOf = (localFlow as any).has_on_behalf_of ?? !!localFlow.on_behalf_of_email; delete (localFlow as any).has_on_behalf_of; + // The authorization half of the identity is never exported to the repo (the + // workspace tarball strips it); it only ever travels back from the remote row. + delete (localFlow as any).on_behalf_of; - const preserveFields: { on_behalf_of_email?: string; preserve_on_behalf_of?: boolean } = {}; + const preserveFields: { + on_behalf_of_email?: string; + on_behalf_of?: string; + preserve_on_behalf_of?: boolean; + } = {}; if (permissionedAsContext?.userIsAdminOrDeployer && hasOnBehalfOf) { if (flow && flow.on_behalf_of_email) { preserveFields.on_behalf_of_email = flow.on_behalf_of_email; + preserveFields.on_behalf_of = (flow as any).on_behalf_of; preserveFields.preserve_on_behalf_of = true; log.info(`Preserving ${flow.on_behalf_of_email} as on_behalf_of for flow ${remotePath}`); } @@ -1218,6 +1226,8 @@ const command = new Command() ...remote, path: flowPath, on_behalf_of_email: email, + // Derived server-side; see the script command for why. + on_behalf_of: undefined, preserve_on_behalf_of: true, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index 7eab79b52d..9680c6abda 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -22,7 +22,7 @@ import { FSFSElement, ignoreF, } from "../sync/sync.ts"; -import { exts } from "../script/script.ts"; +import { hasScriptExt } from "../script/script.ts"; import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint, scriptPathToRemotePath } from "../../utils/resource_folders.ts"; import { listSyncCodebases, SyncCodebase } from "../../utils/codebase.ts"; import { @@ -51,7 +51,7 @@ async function walkLocalScripts( const elems = await elementsToMap( await FSFSElement(process.cwd(), codebases, false), (p, isD) => - (!isD && !exts.some((ext) => p.endsWith(ext))) || + (!isD && !hasScriptExt(p)) || ignore(p, isD) || isFolderResourcePathAnyFormat(p) || // Datatable migration `.sql` files aren't Windmill scripts. @@ -221,7 +221,7 @@ function categorizeLocalFiles( ) { appPaths.push(p); } else if ( - exts.some((ext) => p.endsWith(ext)) && + hasScriptExt(p) && !isFolderResourcePathAnyFormat(p) && // Datatable migration `.sql` files aren't Windmill scripts. !isDatatableMigrationPath(p) && diff --git a/cli/src/commands/gitsync-settings/converter.ts b/cli/src/commands/gitsync-settings/converter.ts index 6861603bb4..868fa1b330 100644 --- a/cli/src/commands/gitsync-settings/converter.ts +++ b/cli/src/commands/gitsync-settings/converter.ts @@ -34,6 +34,7 @@ export class GitSyncSettingsConverter { includeSettings: includeTypes.includes("settings"), includeKey: includeTypes.includes("key"), skipWorkspaceDependencies: !includeTypes.includes("workspacedependencies"), + skipDatatableMigrations: !includeTypes.includes("datatablemigration"), }; // Only include extraIncludes if it has content @@ -63,6 +64,7 @@ export class GitSyncSettingsConverter { if (opts.includeSettings) includeTypes.push("settings"); if (opts.includeKey) includeTypes.push("key"); if (!opts.skipWorkspaceDependencies) includeTypes.push("workspacedependencies"); + if (!opts.skipDatatableMigrations) includeTypes.push("datatablemigration"); const result: BackendGitSyncSettings = { include_path: opts.includes || [], @@ -102,6 +104,7 @@ export class GitSyncSettingsConverter { includeSettings: opts.includeSettings ?? false, includeKey: opts.includeKey ?? false, skipWorkspaceDependencies: opts.skipWorkspaceDependencies ?? false, + skipDatatableMigrations: opts.skipDatatableMigrations ?? false, }; } @@ -126,6 +129,7 @@ export class GitSyncSettingsConverter { includeSettings: opts.includeSettings, includeKey: opts.includeKey, skipWorkspaceDependencies: opts.skipWorkspaceDependencies, + skipDatatableMigrations: opts.skipDatatableMigrations, }; } diff --git a/cli/src/commands/gitsync-settings/types.ts b/cli/src/commands/gitsync-settings/types.ts index 7bebb1f1a1..2a74966ccf 100644 --- a/cli/src/commands/gitsync-settings/types.ts +++ b/cli/src/commands/gitsync-settings/types.ts @@ -38,6 +38,7 @@ export const GIT_SYNC_FIELDS = [ "includeSettings", "includeKey", "skipWorkspaceDependencies", + "skipDatatableMigrations", ] as const; export type GitSyncField = typeof GIT_SYNC_FIELDS[number]; @@ -59,6 +60,7 @@ export const INCLUDE_TYPE_MAPPINGS = { settings: "includeSettings", key: "includeKey", workspacedependencies: "skipWorkspaceDependencies", + datatablemigration: "skipDatatableMigrations", } as const; // Write mode for branch-based configuration diff --git a/cli/src/commands/init/template.ts b/cli/src/commands/init/template.ts index b0c83d7039..ef2bcad0a6 100644 --- a/cli/src/commands/init/template.ts +++ b/cli/src/commands/init/template.ts @@ -97,6 +97,7 @@ export const CONFIG_REFERENCE: ConfigOption[] = [ { name: "skipApps", type: "boolean", default: "false", description: "Skip syncing apps" }, { name: "skipFolders", type: "boolean", default: "false", description: "Skip syncing folders" }, { name: "skipWorkspaceDependencies", type: "boolean", default: "false", description: "Skip syncing workspace dependencies" }, + { name: "skipDatatableMigrations", type: "boolean", default: "false", description: "Skip syncing data table SQL migrations" }, { name: "includeSchedules", type: "boolean", default: "false", description: "Include schedules in sync", commented: true, templateValue: "true", groupNote: "Uncomment to include these (excluded by default):" }, diff --git a/cli/src/commands/pipeline/docs.ts b/cli/src/commands/pipeline/docs.ts index abee64cbf3..1ae38d5e71 100644 --- a/cli/src/commands/pipeline/docs.ts +++ b/cli/src/commands/pipeline/docs.ts @@ -17,11 +17,12 @@ import { colors } from "@cliffy/ansi/colors"; import { GlobalOptions } from "../../types.ts"; import { type AssetGraph, + hideDbtRunnables, buildLocalPipelineGraph, workspaceRoot, } from "./localGraph.ts"; -const ASSET_KINDS = "s3object,ducklake,datatable,volume"; +const ASSET_KINDS = "s3object,ducklake,datatable,volume,dbt"; function assetUri(kind: string, p: string): string { const prefix = kind === "s3object" ? "s3" : kind; @@ -39,7 +40,7 @@ async function fetchDeployedGraph( if (!res.ok) { throw new Error(`GET assets/graph -> ${res.status}: ${await res.text()}`); } - return (await res.json()) as AssetGraph; + return hideDbtRunnables((await res.json()) as AssetGraph); } // Render the pipeline graph as a markdown document. diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index bb51cd98eb..5f9e713f42 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -63,6 +63,10 @@ export type GraphRunnable = { // `buildMacroEdges` (the wasm asset parser emits neither the marker nor the // registry). Non-empty ⇒ definition-only node. macros?: { name: string; params?: string; is_table?: boolean }[]; + // Set by the deployed graph on a dbt script: it owns a whole project, so the + // node counts models rather than reading as a single-output script. Never set + // locally — a dbt descriptor has no asset parser here. + dbt?: { model_count: number }; }; export type GraphEdge = { runnable_kind: string; @@ -229,6 +233,7 @@ function commentPrefix(language: string): string { language === "ansible" || language === "ruby" || language === "rlang" || + language === "dbt" || language === "nu" || language === "powershell" ) @@ -342,7 +347,11 @@ function normalizeRetry(retry: ParseAssetsRaw["retry"]): ParseAssetsRaw["retry"] // Read-asset kinds whose read auto-derives a cascade trigger edge inside a // `// pipeline`. Mirror of backend `is_auto_trigger_kind` (windmill-common // assets.rs) / frontend `AUTO_TRIGGER_KINDS` (resolveGraph.ts) — ducklake -// tables and s3 objects only; resource/datatable/volume stay explicit-`// on`. +// tables and s3 objects; resource/datatable/volume/table stay explicit-`// on`. +// A local graph out of step with that set shows an edge the deploy will not +// cascade along, and the generated pipeline docs then describe the wrong DAG. +// `table` is excluded everywhere: a dbt run does not dispatch, and nothing else +// writes a warehouse relation. const AUTO_TRIGGER_KINDS = new Set(["ducklake", "s3object"]); // Asset-URI prefixes accepted by `// mute `, in lockstep with the @@ -550,6 +559,75 @@ export async function collectScripts( return out; } +// The structural minimum the dbt filter needs. Declared instead of taking +// `AssetGraph` so the bounded-cascade view of the same payload (`BCGraph`, a +// narrower shape over identical JSON) passes through without a cast. +type DbtFilterableGraph = { + runnables: { path: string; usage_kind: string; dbt?: unknown }[]; + edges: { runnable_kind: string; runnable_path: string }[]; + triggers: { runnable_kind: string; runnable_path: string }[]; + macro_edges?: { lib_path: string; consumer_path: string }[]; + test_edges?: { + producer_kind: string; + producer_path: string; + runnable_kind: string; + runnable_path: string; + }[]; +}; + +/** + * The pipeline's view of a deployed graph whose folder also holds a dbt project. + * + * `/assets/graph` is asset-usage driven, not membership driven: it lists every + * script that reads or writes a relation, so a dbt script appears there like any + * producer. That is right for the endpoint — the node is what attributes a + * relation to what builds it — but a dbt project is not a pipeline, so it must + * not be rendered as one of its scripts. Mirrors the frontend's + * `hideDbtRunnables`; the local builder drops these nodes at the source. + * + * Its relations stay: they are what a downstream pipeline script reads. + */ +export function hideDbtRunnables(graph: G): G { + // Keyed by `(usage_kind, path)`, the graph's identity for a runnable — a + // script and a flow may share a path, and keying on path alone would take the + // flow's node, edges and triggers down with the dbt script's. + const key = (usage_kind: string, path: string) => `${usage_kind}:${path}`; + const dbtKeys = new Set( + (graph.runnables ?? []) + .filter((r) => r.dbt) + .map((r) => key(r.usage_kind, r.path)), + ); + if (dbtKeys.size === 0) return graph; + const kept = (usage_kind: string, path: string) => + !dbtKeys.has(key(usage_kind, path)); + return { + ...graph, + runnables: graph.runnables.filter((r) => kept(r.usage_kind, r.path)), + edges: graph.edges.filter((e) => kept(e.runnable_kind, e.runnable_path)), + triggers: graph.triggers.filter((t) => + kept(t.runnable_kind, t.runnable_path), + ), + // A macro library is a script; a dbt script defines no macros, so neither + // end can be a flow. + ...(graph.macro_edges + ? { + macro_edges: graph.macro_edges.filter( + (m) => kept("script", m.lib_path) && kept("script", m.consumer_path), + ), + } + : {}), + ...(graph.test_edges + ? { + test_edges: graph.test_edges.filter( + (t) => + kept(t.producer_kind, t.producer_path) && + kept(t.runnable_kind, t.runnable_path), + ), + } + : {}), + }; +} + // Build the full pipeline asset-graph from local files in `f//`. // Only `// pipeline` scripts become graph nodes (pipeline membership), mirroring // the deployed graph endpoint. Returns the graph plus the in-pipeline scripts' @@ -617,6 +695,11 @@ export async function buildLocalPipelineGraph(args: { }); continue; } + // A dbt project is not a data pipeline: the deploy never marks a dbt script + // `auto_kind='pipeline'` and the pipeline canvas drops its node, so the + // local graph must not invent one. Its models don't belong here either — + // they come from the manifest the deploy derives, not from the descriptor. + if (s.language === "dbt") continue; if (!out.in_pipeline) continue; // not a pipeline member if (out.data_tests && out.data_tests.length > 0) { dataTestsByPath.set(s.path, out.data_tests); diff --git a/cli/src/commands/pipeline/pipeline.ts b/cli/src/commands/pipeline/pipeline.ts index 0738283fba..8d46119e14 100644 --- a/cli/src/commands/pipeline/pipeline.ts +++ b/cli/src/commands/pipeline/pipeline.ts @@ -24,6 +24,7 @@ import { } from "./boundedCascade.ts"; import { type AssetGraph, + hideDbtRunnables, type GraphTrigger, type LocalScript, buildLocalPipelineGraph, @@ -96,7 +97,7 @@ async function list(opts: GlobalOptions & { json?: boolean }) { } } -const ASSET_KINDS = "s3object,ducklake,datatable,volume"; +const ASSET_KINDS = "s3object,ducklake,datatable,volume,dbt"; function assetUri(kind: string, path: string): string { const prefix = kind === "s3object" ? "s3" : kind; @@ -132,8 +133,10 @@ async function show( } else { const workspace = await resolveWorkspace(opts); await requireLogin(opts); - graph = await apiGet( - `/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f)}&asset_kinds=${ASSET_KINDS}`, + graph = hideDbtRunnables( + await apiGet( + `/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f)}&asset_kinds=${ASSET_KINDS}`, + ), ); enrich = (nativeByScript, roots) => enrichRootMarkers(workspace.workspaceId, graph, nativeByScript, roots); } @@ -612,8 +615,10 @@ async function run( } } } else { - graph = await apiGet( - `/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f)}&asset_kinds=${ASSET_KINDS}`, + graph = hideDbtRunnables( + await apiGet( + `/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f)}&asset_kinds=${ASSET_KINDS}`, + ), ); // Recover marker-only `data_upload`/`webhook`/`email` triggers the graph // endpoint can't emit, so input-only entrypoints are cut here as they are diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index c699e3d58f..34a2b09949 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -71,10 +71,19 @@ import { isScriptModulePath, buildModuleFolderPath, getModuleFolderSuffix, + dbtGeneratedDirs, + isUnderGeneratedDir, + isLocalSecretFile, + moduleFileExclusion, + oversizedModuleFileError, + MAX_MODULE_BYTES, isModuleEntryPoint, getScriptBasePathFromModulePath, scriptPathToRemotePath, isRawAppPath, + DBT_DESCRIPTOR_NAME, + isDbtDescriptorPath, + isMissingDbtDescriptor, } from "../../utils/resource_folders.ts"; export interface ScriptFile { @@ -159,9 +168,18 @@ async function push(opts: PushOptions, filePath: string) { return; } - const fstat = await stat(filePath); - if (!fstat.isFile()) { - throw new Error("file path must refer to a file."); + // A dbt project's descriptor is optional, so the one content path a + // descriptor-less project has is deliberately not on disk. The project beside + // it is what says the script is real. + const absentDescriptor = await stat(filePath).then( + () => false, + (e) => isMissingDbtDescriptor(filePath, e) + ); + if (!absentDescriptor) { + const fstat = await stat(filePath); + if (!fstat.isFile()) { + throw new Error("file path must refer to a file."); + } } if (filePath.endsWith(".script.json") || filePath.endsWith(".script.yaml")) { @@ -174,7 +192,7 @@ async function push(opts: PushOptions, filePath: string) { // Warn about metadata state before pushing try { - const content = await readTextFile(filePath); + const content = await readScriptContent(filePath); const remotePath = removeExtensionToPath(filePath).replaceAll(SEP, "/"); const contentHash = await computePushMetadataHash(filePath, content); const conf = await readLockfile(); @@ -272,7 +290,7 @@ const MODULE_ENTRY_META_RE = /([\\/])script\.(yaml|json|lock)$/; * `__mod/`, so this only narrows it to the metadata extensions: a `script.yaml` * nested deeper in the module tree is a module file, not the script's metadata. */ -function isModuleEntryMetadata(p: string): boolean { +export function isModuleEntryMetadata(p: string): boolean { return isModuleEntryPoint(p) && MODULE_ENTRY_META_RE.test(p); } @@ -335,7 +353,7 @@ export async function handleFile( // standalone scripts — pushed via pushRawApp, not here. !isRawAppPath(path) && (!isScriptModulePath(path) || moduleEntryPoint) && - exts.some((exts) => path.endsWith(exts)) + hasScriptExt(path) ) { if (alreadySynced.includes(path)) { return true; @@ -345,6 +363,29 @@ export async function handleFile( alreadySynced.push(path); const remotePath = scriptPathToRemotePath(path); + // Before anything is written: `.py` and `__dbt/` deploy to ONE + // remote path, so whichever is pushed last replaces the other's script. + // Refused from either side — the descriptor is exempt only from finding its + // OWN project (it is that project's content file, so its base resolves to + // the same `dbt_project.yml`), never from an ordinary sibling. + // A folder-layout script is `__mod/script.ts`, so stripping its + // extension yields `__mod/script`, not the base both layouts deploy + // to. Wrong base, and the probe below looks in a directory that cannot + // exist — which is how a `__mod` script and a dbt project at one path were + // both pushed, each replacing the other. + const base = isScriptModulePath(path) + ? getScriptBasePathFromModulePath(path) ?? removeExtensionToPath(path) + : removeExtensionToPath(path); + const isDescriptor = isDbtDescriptorPath(path); + const other = isDescriptor + ? await collidingOrdinaryScript(base) + : await collidingDbtProject(base); + if (other) { + throw isDescriptor + ? dbtPathCollisionError(path, other) + : dbtPathCollisionError(other, path); + } + const language = inferContentTypeFromFilePath(path, opts?.defaultTs); const codebase = @@ -479,7 +520,7 @@ export async function handleFile( } catch { log.debug(`Script ${remotePath} does not exist on remote`); } - const content = await readTextFile(path); + const content = await readScriptContent(path); if (opts?.skipScriptsMetadata) { // if (codebase) { @@ -499,8 +540,14 @@ export async function handleFile( const scriptBasePath = moduleEntryPoint ? getScriptBasePathFromModulePath(path)! : path.substring(0, path.indexOf(".")); - const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(); - const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, moduleEntryPoint); + const isDbt = language === "dbt"; + const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(language); + const modules = await readModulesFromDisk( + moduleFolderPath, + opts?.defaultTs, + moduleEntryPoint, + isDbt, + ); // A concurrent_limit of <= 0 means "concurrency disabled", not "zero slots" (which // would brick the runnable at the queue's concurrency gate). Emit it as omitted rather @@ -518,7 +565,11 @@ export async function handleFile( path: remotePath.replaceAll(SEP, "/"), summary: typed?.summary ?? "", kind: typed?.kind, - lock: typed?.lock, + // A dbt lock pins a resolved commit and engine versions that only a + // dependency job can determine, and that job is also what publishes the + // script's manifest graph. Sending one suppresses that job, so the push + // would deploy a stale lock AND leave the graph unpublished. + lock: language === "dbt" ? undefined : typed?.lock, schema: typed?.schema, tag: typed?.tag, ws_error_handler_muted: typed?.ws_error_handler_muted, @@ -544,10 +595,16 @@ export async function handleFile( const hasOnBehalfOf = (typed as any)?.has_on_behalf_of ?? !!typed?.on_behalf_of_email; delete (typed as any)?.has_on_behalf_of; + // The authorization half of the identity is never exported to the repo (the + // workspace tarball strips it); it only ever travels back from the remote row. + delete (typed as any)?.on_behalf_of; if (permissionedAsContext?.userIsAdminOrDeployer && hasOnBehalfOf) { if (remote && remote.on_behalf_of_email) { requestBodyCommon.on_behalf_of_email = remote.on_behalf_of_email; + (requestBodyCommon as any).on_behalf_of = ( + remote as any + ).on_behalf_of; (requestBodyCommon as any).preserve_on_behalf_of = true; log.info(`Preserving ${remote.on_behalf_of_email} as on_behalf_of for script ${remotePath}`); } @@ -679,6 +736,11 @@ export async function readModulesFromDisk( moduleFolderPath: string, defaultTs: "bun" | "deno" | undefined, folderLayout: boolean = false, + // A dbt project rides in its module folder as-is: `.sql` models (which the + // language inference below rejects as an ambiguous dialect), `.yml` schemas + // and `.csv` seeds are all part of the project and none is a Windmill script. + // Verbatim, or dbt receives a project missing exactly the files it needs. + verbatim: boolean = false, ): Promise | undefined> { if (!fs.existsSync(moduleFolderPath) || !fs.statSync(moduleFolderPath).isDirectory()) { return undefined; @@ -686,9 +748,18 @@ export async function readModulesFromDisk( const modules: Record = {}; + const skipDirs = verbatim + ? dbtGeneratedDirs(moduleFolderPath) + : new Set(); + // In folder layout mode, skip the entry point files (script.*, script.yaml, etc.) const isEntryPointFile = (name: string, isTopLevel: boolean) => { - if (!folderLayout || !isTopLevel) return false; + if (!isTopLevel) return false; + // A dbt project's descriptor is the script's CONTENT, so it must not also + // ride along as a module: the push would send the same text twice and dbt + // would find a stray file at its project root. + if (verbatim) return name === DBT_DESCRIPTOR_NAME; + if (!folderLayout) return false; return ( name.startsWith("script.") || name === "script.lock" || @@ -705,10 +776,63 @@ export async function readModulesFromDisk( const isTopLevel = relPrefix === ""; if (entry.isDirectory()) { + // A configured `target-path` may be nested (`build/target`), so the + // comparison is on the project-relative path, not the entry name. + if (skipDirs.size > 0 && isUnderGeneratedDir(relPath, skipDirs)) continue; readDir(fullPath, relPath); - } else if (entry.isFile() && !entry.name.endsWith(".lock") && !isEntryPointFile(entry.name, isTopLevel)) { - // Skip lock files — they're handled as the `lock` field on ScriptModule - if (exts.some((ext) => entry.name.endsWith(ext))) { + // `.lock` is the script's own lockfile in a `__mod` bundle (the `lock` + // field on ScriptModule) — but a dbt project's files are its author's, + // and one may legitimately be named `uv.lock`. Dropping it would break + // the unmodified-project round trip this bundle exists to keep. + } else if ( + entry.isFile() && + (verbatim || !entry.name.endsWith(".lock")) && + !isEntryPointFile(entry.name, isTopLevel) + ) { + if (verbatim) { + // Secrets stay on the machine that holds them. Skipped before the + // read, and loudly: a `.env` swept into the bundle is a credential + // stored in every version of the script and handed back on pull. + if (isLocalSecretFile(entry.name)) { + log.warn( + `Skipping ${relPath}: a local secrets file is not part of the dbt project — ` + + `dbt reads its values from the environment, so set them in the script's ` + + `environment variables or the descriptor's \`env\``, + ); + continue; + } + // A dbt project's authored files are text. A binary one -- an image + // under `docs/`, a `.DS_Store`, a parquet seed -- would be read as + // mojibake and, if it carries a NUL, rejected by Postgres with an + // opaque `unsupported Unicode escape sequence`, which the push then + // reports as success. Skip it, loudly: dbt does not read it either. + // + // Asked BEFORE reading: the predicate only stats the file and reads + // its first 8 KB, so a multi-gigabyte seed next to the project costs + // that rather than being loaded whole just to be rejected. + const exclusion = moduleFileExclusion(fullPath); + if (exclusion !== undefined) { + // Over the limit but readable as text — a large seed CSV is the + // realistic case — is refused rather than skipped: dbt WOULD have + // read it, so shipping the project without it deploys something that + // compiles here and fails at run time with a missing relation. + if (exclusion === "oversized") { + throw oversizedModuleFileError(relPath, fs.statSync(fullPath).size); + } + log.warn( + `Skipping ${relPath}: not a text file, so it is not part of the dbt project the ` + + `bundle carries — dbt does not read it either`, + ); + continue; + } + // `language` is a required field of the API type and is not used for + // these: the worker writes them to their relative path and dbt reads + // the tree. + modules[relPath] = { + content: fs.readFileSync(fullPath).toString("utf-8"), + language: "dbt" as ScriptModule["language"], + }; + } else if (exts.some((ext) => entry.name.endsWith(ext))) { const content = readTextFileSync(fullPath); const language = inferContentTypeFromFilePath(entry.name, defaultTs); @@ -889,6 +1013,90 @@ async function createScript( */ export class UnresolvableScriptContentFileError extends Error {} +/** + * A path claimed by both a dbt project and an ordinary script. + * + * Its own class because the module push tolerates "no parent found" and must + * NOT tolerate this: swallowed, the command reports success while deploying + * nothing. + */ +export class DbtPathCollisionError extends UnresolvableScriptContentFileError {} + +/** + * The dbt project a path would collide with, if there is one. + * + * `.py` and `__dbt/` deploy to the SAME remote path, so whichever + * is pushed last wins and replaces the other's script. The descriptor is + * optional, so `dbt_project.yml` — not the descriptor — is what says a project + * is there. Asked on BOTH push paths: an ordinary file goes straight to + * `handleFile`, a model reaches its parent through `findContentFile`, and a + * guard on one of them leaves the other silently overwriting. + */ +export async function collidingDbtProject( + basePath: string +): Promise { + const project = basePath + "__dbt/dbt_project.yml"; + return (await stat(project).then(() => true).catch(() => false)) + ? project + : undefined; +} + +/** + * The ordinary script file sharing a base with a dbt project, if there is one — + * the same collision as [`collidingDbtProject`], seen from the dbt side. + * + * Needed because a descriptor may be pushed DIRECTLY (`wmill script push + * __dbt/wm_dbt.yaml`), which never passes through the metadata resolution + * that would otherwise catch it. + */ +export async function collidingOrdinaryScript( + basePath: string +): Promise { + for (const ext of exts) { + if (ext === "__dbt/" + DBT_DESCRIPTOR_NAME) continue; + // Both layouts, because both deploy to `basePath`: the flat file, and the + // folder layout's entry point. + for (const candidate of [ + basePath + ext, + `${basePath}${getModuleFolderSuffix()}/script${ext}`, + ]) { + const isFile = await stat(candidate) + .then((s) => s.isFile()) + .catch(() => false); + if (isFile) return candidate; + } + } + return undefined; +} + +export function dbtPathCollisionError( + project: string, + other: string +): DbtPathCollisionError { + return new DbtPathCollisionError( + `${project} and ${other} deploy to the same path, so pushing either one ` + + `replaces the other's script. Keep one: move the dbt project to a path ` + + `of its own, or remove ${other}.` + ); +} + + +/** + * A script's content, tolerating the one content file that may not exist: a dbt + * project's descriptor is optional, and absent means an empty descriptor. + */ +async function readScriptContent(filePath: string): Promise { + try { + return await readTextFile(filePath); + } catch (e) { + // ONLY a missing file is an empty descriptor. A permission or I/O error on a + // descriptor that does exist would otherwise deploy the defaults — the + // `main` warehouse and the whole project — in place of what the file says. + if (isMissingDbtDescriptor(filePath, e)) return ""; + throw e; + } +} + export async function findContentFile(filePath: string) { // Folder layout: __mod/script.yaml -> __mod/script.ts const isModuleFolderMeta = isModuleEntryMetadata(filePath); @@ -924,6 +1132,20 @@ export async function findContentFile(filePath: string) { ) .filter((x) => x.file) .map((x) => x.path); + // A dbt project's descriptor is OPTIONAL, so `dbt_project.yml` is what says a + // dbt script lives at this path — the descriptor is often absent from the + // candidates above while the project is perfectly real. Asked BEFORE the + // counts below: a project beside an ordinary script is not "one candidate", + // it is two scripts claiming one remote path, and returning the ordinary one + // deploys it OVER the dbt script on the next push of any model. + const dbtCandidate = toCandidate("__dbt/" + DBT_DESCRIPTOR_NAME); + const dbtProject = await collidingDbtProject( + dbtCandidate.slice(0, -("__dbt/" + DBT_DESCRIPTOR_NAME).length), + ); + const nonDbtCandidates = validCandidates.filter((c) => c !== dbtCandidate); + if (dbtProject && nonDbtCandidates.length > 0) { + throw dbtPathCollisionError(dbtProject, nonDbtCandidates.join(", ")); + } if (validCandidates.length > 1) { throw new UnresolvableScriptContentFileError( `Multiple script files found next to ${filePath}: ${validCandidates.join(", ")} — ` + @@ -931,6 +1153,11 @@ export async function findContentFile(filePath: string) { ); } if (validCandidates.length < 1) { + // Resolving to the absent descriptor keeps one content path for every + // caller; reading it yields an empty descriptor. + if (dbtProject) { + return dbtCandidate; + } throw new UnresolvableScriptContentFileError( `No script file found next to ${filePath} — a script cannot be deployed from its metadata alone. ` + `Add the matching script file (e.g. ${toCandidate(".ts")} or ${toCandidate( @@ -999,6 +1226,10 @@ export function filePathExtensionFromContentType( return ".rb"; } else if (language === "rlang") { return ".r"; + } else if (language === "dbt") { + // Not an extension but a path suffix: a dbt script's content file lives + // inside the project folder, so ` + this` is where it belongs. + return "__dbt/" + DBT_DESCRIPTOR_NAME; // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); @@ -1031,12 +1262,28 @@ export const exts = [ ".java", ".rb", ".r", + // Not an extension: a dbt script's content file is its descriptor, inside + // the project folder. `.script.yaml` -> `__dbt/wm_dbt.yaml`. + "__dbt/" + DBT_DESCRIPTOR_NAME, // for related places search: ADD_NEW_LANG ]; +/** + * Whether a path is a script's content file. + * + * Separators are normalized first: one "extension" is the path suffix + * `__dbt/wm_dbt.yaml`, which on Windows is spelled `__dbt\wm_dbt.yaml` and + * would match nothing — silently skipping every dbt project on that platform. + */ +export function hasScriptExt(p: string): boolean { + const norm = p.replaceAll("\\", "/"); + return exts.some((ext) => norm.endsWith(ext)); +} + export function removeExtensionToPath(path: string): string { + const norm = path.replaceAll("\\", "/"); for (const ext of exts) { - if (path.endsWith(ext)) { + if (norm.endsWith(ext)) { return path.substring(0, path.length - ext.length); } } @@ -1450,7 +1697,7 @@ export async function generateMetadata( await FSFSElement(process.cwd(), codebases, false), (p, isD) => { return ( - (!isD && !exts.some((ext) => p.endsWith(ext))) || + (!isD && !hasScriptExt(p)) || ignore(p, isD) || isFlowPath(p) || isAppPath(p) || @@ -1569,9 +1816,17 @@ async function preview( return; } - const fstat = await stat(filePath); - if (!fstat.isFile()) { - throw new Error("file path must refer to a file."); + // Same as push: a descriptor-less dbt project's content path is deliberately + // absent, and the project beside it is what says the script is real. + const absentDescriptor = await stat(filePath).then( + () => false, + (e) => isMissingDbtDescriptor(filePath, e) + ); + if (!absentDescriptor) { + const fstat = await stat(filePath); + if (!fstat.isFile()) { + throw new Error("file path must refer to a file."); + } } if (filePath.endsWith(".script.json") || filePath.endsWith(".script.yaml")) { @@ -1582,15 +1837,23 @@ async function preview( const codebases = await listSyncCodebases(opts); const language = inferContentTypeFromFilePath(filePath, opts?.defaultTs); - const content = await readTextFile(filePath); + const content = await readScriptContent(filePath); const input = opts.data ? await resolve(opts.data) : {}; - // Read modules from __mod/ folder if present + // Read modules from the bundle folder if present. Same suffix and same + // verbatim read as deploy: a dbt project lives in `__dbt/`, and parsing its + // files as scripts would drop the `dbt_project.yml` the executor looks for. const isFolderLayout = isModuleEntryPoint(filePath); + const isDbt = language === "dbt"; const moduleFolderPath = isFolderLayout ? path.dirname(filePath) - : filePath.substring(0, filePath.indexOf(".")) + getModuleFolderSuffix(); - const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, isFolderLayout); + : filePath.substring(0, filePath.indexOf(".")) + getModuleFolderSuffix(language); + const modules = await readModulesFromDisk( + moduleFolderPath, + opts?.defaultTs, + isFolderLayout, + isDbt + ); // Check if this is a codebase script const codebase = @@ -1872,6 +2135,10 @@ async function setPermissionedAs( lock: Array.isArray(remote.lock) ? remote.lock.join("\n") : remote.lock ?? undefined, parent_hash: remote.hash, on_behalf_of_email: email, + // The principal is derived server-side from the email, which resolves workspace + // members, groups and superadmins acting outside their workspaces alike — a + // client-side `usr` lookup would see only the first of those. + on_behalf_of: undefined, preserve_on_behalf_of: true, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 90a31f4919..2702e13714 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -84,7 +84,9 @@ export async function downloadZip( includeSettings?: boolean, includeKey?: boolean, skipWorkspaceDependencies?: boolean, - defaultTs?: "bun" | "deno" + skipDatatableMigrations?: boolean, + defaultTs?: "bun" | "deno", + syncBehavior?: string ): Promise { const requestHeaders = new Headers(); requestHeaders.set("Authorization", "Bearer " + workspace.token); @@ -98,6 +100,9 @@ export async function downloadZip( } const includeWorkspaceDependenciesValue = !(skipWorkspaceDependencies ?? false); + // `sync_behavior_version` lets the server skip work this client would only throw away: + // from v1 the on-behalf-of address is stripped below, so the tarball sends the + // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs // on flow / script / app rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities @@ -107,7 +112,7 @@ export async function downloadZip( }&skip_secrets=${skipSecrets ?? false}&include_schedules=${includeSchedules ?? false }&include_triggers=${includeTriggers ?? false}&include_users=${includeUsers ?? false }&include_groups=${includeGroups ?? false}&include_settings=${includeSettings ?? false - }&include_key=${includeKey ?? false}&include_workspace_dependencies=${includeWorkspaceDependenciesValue}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}&settings_version=v2&preserve_extra_perms=true`; + }&include_key=${includeKey ?? false}&include_workspace_dependencies=${includeWorkspaceDependenciesValue}&skip_datatable_migrations=${skipDatatableMigrations ?? false}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}&settings_version=v2&preserve_extra_perms=true&sync_behavior_version=${syncBehavior ?? "v0"}`; const baseUrl = workspace.remote + "api/w/" + workspace.workspaceId + "/workspaces/tarball?"; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 3e6dd464fc..196c4c54e8 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -1,13 +1,27 @@ import { requireLogin } from "../../core/auth.ts"; import { fetchVersion, resolveWorkspace } from "../../core/context.ts"; -import { writeFile, readdir, stat, rm, copyFile, mkdir } from "node:fs/promises"; +import { + writeFile, + readdir, + stat, + rm, + copyFile, + mkdir, +} from "node:fs/promises"; +import { existsSync } from "node:fs"; import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import * as log from "../../core/log.ts"; import * as path from "node:path"; import { sep as SEP } from "node:path"; -import { stringify as yamlStringify, type DocumentOptions, type SchemaOptions, type CreateNodeOptions, type ToStringOptions } from "yaml"; +import { + stringify as yamlStringify, + type DocumentOptions, + type SchemaOptions, + type CreateNodeOptions, + type ToStringOptions, +} from "yaml"; import JSZip from "jszip"; import { minimatch } from "minimatch"; import { yamlParseContent } from "../../utils/yaml.ts"; @@ -39,13 +53,15 @@ import { exts, findContentFile, findResourceFile, + isModuleEntryMetadata, handleScriptMetadata, UnresolvableScriptContentFileError, removeExtensionToPath, filePathExtensionFromContentType, + hasScriptExt, } from "../script/script.ts"; -import { handleFile } from "../script/script.ts"; +import { DbtPathCollisionError, handleFile } from "../script/script.ts"; import { deepEqual, fetchRemoteVersion, @@ -86,7 +102,9 @@ import { gitSyncDeployPush, deriveGitSyncDeployIncludes, isForkWorkspace, + gitRecordedDatatableMigrationPaths, type GitSyncDeployItem, + type RecordedMigrationPaths, } from "../../utils/git.ts"; import { Workspace } from "../workspace/workspace.ts"; import { removePathPrefix } from "../../types.ts"; @@ -99,20 +117,31 @@ import { MalformedLockfileError, workspaceDependenciesPathToLanguageAndFilename, } from "../../utils/metadata.ts"; -import { DoubleLinkedDependencyTree, uploadScripts } from "../../utils/dependency_tree.ts"; -import { OpenFlow, NativeServiceName, ScriptModule } from "../../../gen/types.gen.ts"; +import { + DoubleLinkedDependencyTree, + uploadScripts, +} from "../../utils/dependency_tree.ts"; +import { + OpenFlow, + NativeServiceName, + ScriptModule, +} from "../../../gen/types.gen.ts"; import { pushResource } from "../resource/resource.ts"; import { newPathAssigner, newRawAppPathAssigner, PathAssigner, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; -import { extractInlineScripts as extractInlineScriptsForFlows, extractCurrentMapping } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; +import { + extractInlineScripts as extractInlineScriptsForFlows, + extractCurrentMapping, +} from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; import { isExecutionModeAnonymous } from "../app/app.ts"; import { APP_BACKEND_FOLDER, generateAppLocksInternal, + RECORDINGS_FOLDER, } from "../app/app_metadata.ts"; import { isFlowPath, @@ -132,10 +161,15 @@ import { getFolderSuffixWithSep, getNonDottedPaths, isScriptModulePath, + oversizedDbtFileError, getModuleFolderSuffix, + isDbtModulePath, + isDbtGeneratedPath, isModuleEntryPoint, getScriptBasePathFromModulePath, hasWrongFormatSuffix, + DBT_DESCRIPTOR_NAME, + isDbtDescriptorPath, } from "../../utils/resource_folders.ts"; let branchDeprecationWarned = false; @@ -154,12 +188,12 @@ function configKeyForItemKind( return "resources"; case "variable": return "variables"; - // case "schedule": - // return "schedules"; - // default: - // return kind.endsWith("_trigger") ? "triggers" : null; - } - return null + // case "schedule": + // return "schedules"; + // default: + // return kind.endsWith("_trigger") ? "triggers" : null; + } + return null; } // Fetch ws_specific items from the server and merge their paths into specificItems. @@ -182,8 +216,11 @@ async function mergeWsSpecificFromServer( // 404 = endpoint not present on an older server: expected, log at debug. // Anything else (401/403/network) is a real failure that produces an // incomplete sync — surface it so the user notices. - const isApiError = err && typeof err === "object" && - "name" in err && (err as { name: unknown }).name === "ApiError"; + const isApiError = + err && + typeof err === "object" && + "name" in err && + (err as { name: unknown }).name === "ApiError"; const status = isApiError ? (err as { status?: number }).status : undefined; if (status === 404) { log.debug("listWsSpecific endpoint not available on server, skipping"); @@ -235,9 +272,7 @@ export function computeWsSpecificFlagOnlyPushes( ): Array<{ kind: string; serverPath: string; filePath: string }> { if (!localSpecificItems || serverItems === null) return []; - const serverSet = new Set( - serverItems.map((i) => `${i.item_kind}:${i.path}`), - ); + const serverSet = new Set(serverItems.map((i) => `${i.item_kind}:${i.path}`)); const out: Array<{ kind: string; serverPath: string; filePath: string }> = []; for (const filePath of Object.keys(localMap)) { @@ -261,7 +296,10 @@ export function computeWsSpecificFlagOnlyPushes( // Resolve workspace name from a --branch override (git branch → workspace name). // Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch(opts: SyncOptions, branchName: string): string { +function resolveWsNameFromBranch( + opts: SyncOptions, + branchName: string, +): string { const match = findWorkspaceByGitBranch(opts.workspaces, branchName); return match ? match[0] : branchName; } @@ -288,17 +326,23 @@ export function resolveWsNameForConfigFromFlags( } // Warn if --workspace overrides auto-detected branch or if workspace not in config. -function warnWorkspaceOverride(opts: SyncOptions, wsNameForConfig: string | undefined): void { +function warnWorkspaceOverride( + opts: SyncOptions, + wsNameForConfig: string | undefined, +): void { if (!wsNameForConfig || !opts.workspaces) return; // Check if workspace exists in config - const wsEntry = (opts.workspaces as any)?.[wsNameForConfig] as WorkspaceEntryConfig | undefined; + const wsEntry = (opts.workspaces as any)?.[wsNameForConfig] as + WorkspaceEntryConfig | undefined; if (!wsEntry) { - const wsNames = Object.keys(opts.workspaces).filter((k) => k !== "commonSpecificItems"); + const wsNames = Object.keys(opts.workspaces).filter( + (k) => k !== "commonSpecificItems", + ); if (wsNames.length > 0) { log.warn( `⚠️ Workspace '${wsNameForConfig}' is not defined in the 'workspaces' section of wmill.yaml.\n` + - ` No workspace-specific overrides will be applied. Available workspaces: ${wsNames.join(", ")}` + ` No workspace-specific overrides will be applied. Available workspaces: ${wsNames.join(", ")}`, ); } return; @@ -308,11 +352,14 @@ function warnWorkspaceOverride(opts: SyncOptions, wsNameForConfig: string | unde if (isGitRepository()) { const currentBranch = getCurrentGitBranch(); if (currentBranch) { - const autoMatch = findWorkspaceByGitBranch(opts.workspaces, currentBranch); + const autoMatch = findWorkspaceByGitBranch( + opts.workspaces, + currentBranch, + ); if (autoMatch && autoMatch[0] !== wsNameForConfig) { log.info( `Current git branch '${currentBranch}' maps to workspace '${autoMatch[0]}', ` + - `but --workspace overrides to '${wsNameForConfig}'.` + `but --workspace overrides to '${wsNameForConfig}'.`, ); } } @@ -327,9 +374,14 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { // After resolveWorkspace, infer the workspace config name from the resolved profile // by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile(opts: SyncOptions, profile: { remote: string; workspaceId: string }): string | undefined { +function inferWsNameFromProfile( + opts: SyncOptions, + profile: { remote: string; workspaceId: string }, +): string | undefined { if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter((k) => k !== "commonSpecificItems"); + const wsNames = Object.keys(opts.workspaces).filter( + (k) => k !== "commonSpecificItems", + ); for (const name of wsNames) { const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; if (!entry?.baseUrl) continue; @@ -362,7 +414,13 @@ async function resolveEffectiveSyncOptions( promotion?: string, workspaceNameOverride?: string, ): Promise { - return await getEffectiveSettings(localConfig, promotion, false, false, workspaceNameOverride); + return await getEffectiveSettings( + localConfig, + promotion, + false, + false, + workspaceNameOverride, + ); } type DynFSElement = { @@ -462,6 +520,27 @@ async function addCodebaseDigestIfRelevant( return content; } +/** + * Whether a script's modules ARE a dbt project. + * + * Keyed on `dbt_project.yml` rather than on the descriptor: the descriptor is + * optional, so its absence says nothing, while a dbt project without + * `dbt_project.yml` is one dbt itself refuses to run. + * + * Its LANGUAGE decides, not its name. A dbt bundle is read verbatim and every + * file in it is stored as `dbt`; an ordinary modular script that happens to + * vendor a dbt project stores that same file as whatever its extension infers, + * and calling it dbt would lay the bundle out as `__dbt` and drop it on the + * next push. + */ +function isDbtModules(modules: unknown): boolean { + if (typeof modules !== "object" || modules === null) return false; + const marker = (modules as Record)[ + "dbt_project.yml" + ]; + return marker?.language === "dbt"; +} + export async function FSFSElement( p: string, codebases: SyncCodebase[], @@ -491,8 +570,14 @@ export async function FSFSElement( } }, async getContentText(): Promise { - const content = await readTextFile(localP); const itemPath = localP.substring(p.length + 1); + // BEFORE the read: an oversized dbt project file stays visible to the + // diff on purpose (so the push reports it rather than silently shipping + // an incomplete project), and buffering a multi-gigabyte seed to reach + // that error is what this refusal exists to avoid. + const oversized = oversizedDbtFileError(localP, itemPath); + if (oversized) throw oversized; + const content = await readTextFile(localP); const r = await addCodebaseDigestIfRelevant( itemPath, content, @@ -525,9 +610,14 @@ function prioritizeName(name: string): string { return name; } -export const yamlOptions: DocumentOptions & SchemaOptions & CreateNodeOptions & ToStringOptions = { +export const yamlOptions: DocumentOptions & + SchemaOptions & + CreateNodeOptions & + ToStringOptions = { sortMapEntries: (a, b) => { - return prioritizeName(String(a.key)).localeCompare(prioritizeName(String(b.key))); + return prioritizeName(String(a.key)).localeCompare( + prioritizeName(String(b.key)), + ); }, aliasDuplicateObjects: false, singleQuote: true, @@ -591,11 +681,15 @@ export function extractFieldsForRawApps(runnables: Record) { * References the raw-app skill for complete documentation and includes instance-specific * data configuration (datatable, schema, whitelisted tables). */ -export function generateAgentsDocumentation(data: { - tables?: string[]; - datatable?: string; - schema?: string; -} | undefined): string { +export function generateAgentsDocumentation( + data: + | { + tables?: string[]; + datatable?: string; + schema?: string; + } + | undefined, +): string { const tables = data?.tables ?? []; const defaultDatatable = data?.datatable; const defaultSchema = data?.schema; @@ -610,15 +704,19 @@ This file contains **app-specific configuration** for this raw app instance. ## Data Configuration -${defaultDatatable - ? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ''}` - : '**No default datatable configured.** Set \`data.datatable\` in \`raw_app.yaml\` to enable database access.'} +${ + defaultDatatable + ? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ""}` + : "**No default datatable configured.** Set \`data.datatable\` in \`raw_app.yaml\` to enable database access." +} ### Whitelisted Tables -${tables.length > 0 - ? `These tables are accessible to this app:\n\n${tables.map(t => `- \`${t}\``).join('\n')}` - : `**No tables whitelisted.** Add tables to \`data.tables\` in \`raw_app.yaml\`.`} +${ + tables.length > 0 + ? `These tables are accessible to this app:\n\n${tables.map((t) => `- \`${t}\``).join("\n")}` + : `**No tables whitelisted.** Add tables to \`data.tables\` in \`raw_app.yaml\`.` +} ### Adding a Table @@ -626,10 +724,10 @@ Edit \`raw_app.yaml\`: \`\`\`yaml data: - datatable: ${defaultDatatable || 'main'} - ${defaultSchema ? `schema: ${defaultSchema}\n ` : ''}tables: -${tables.length > 0 ? tables.map(t => ` - ${t}`).join('\n') : ' # Add tables here'} - - ${defaultDatatable || 'main'}/${defaultSchema ? defaultSchema + ':' : ''}new_table # ← Add like this + datatable: ${defaultDatatable || "main"} + ${defaultSchema ? `schema: ${defaultSchema}\n ` : ""}tables: +${tables.length > 0 ? tables.map((t) => ` - ${t}`).join("\n") : " # Add tables here"} + - ${defaultDatatable || "main"}/${defaultSchema ? defaultSchema + ":" : ""}new_table # ← Add like this \`\`\` **Table reference formats:** @@ -665,11 +763,15 @@ const rows = await sql\`SELECT * FROM table WHERE id = \${id}\`.fetch(); * Generates a simple DATATABLES.md with just the current configuration summary. * The detailed schema information is generated by generate_datatables.ts command. */ -export function generateDatatablesDocumentation(data: { - tables?: string[]; - datatable?: string; - schema?: string; -} | undefined): string { +export function generateDatatablesDocumentation( + data: + | { + tables?: string[]; + datatable?: string; + schema?: string; + } + | undefined, +): string { const tables = data?.tables ?? []; const defaultDatatable = data?.datatable; const defaultSchema = data?.schema; @@ -683,15 +785,19 @@ Run \`wmill app generate-agents\` to refresh with current workspace schemas. ## Current Configuration -${defaultDatatable - ? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ''}` - : '**No default datatable configured.**'} +${ + defaultDatatable + ? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ""}` + : "**No default datatable configured.**" +} ## Whitelisted Tables -${tables.length > 0 - ? `${tables.map(t => `- \`${t}\``).join('\n')}` - : `*No tables whitelisted. Add tables to \`data.tables\` in \`raw_app.yaml\`.*`} +${ + tables.length > 0 + ? `${tables.map((t) => `- \`${t}\``).join("\n")}` + : `*No tables whitelisted. Add tables to \`data.tables\` in \`raw_app.yaml\`.*` +} --- @@ -761,11 +867,17 @@ export function extractInlineScriptsForApps( return []; } -type FileResourceTypeInfo = { format_extension: string | null; is_fileset: boolean }; +type FileResourceTypeInfo = { + format_extension: string | null; + is_fileset: boolean; +}; function parseFileResourceTypeMap( raw: Record, -): { formatExtMap: Record; filesetMap: Record } { +): { + formatExtMap: Record; + filesetMap: Record; +} { const formatExtMap: Record = {}; const filesetMap: Record = {}; for (const [k, v] of Object.entries(raw)) { @@ -799,7 +911,9 @@ async function findFilesetResourceFile(changePath: string): Promise { // not found, try next } } - throw new Error(`No resource metadata file found for fileset resource: ${changePath}`); + throw new Error( + `No resource metadata file found for fileset resource: ${changePath}`, + ); } type FilesetPushResult = @@ -871,9 +985,13 @@ function ZipFSElement( const content = await zip.files[filename].async("text"); const parsed = JSON.parse(content); if (parsed.modules && Object.keys(parsed.modules).length > 0) { - _moduleScriptPaths.add( - filename.slice(0, -".script.json".length) - ); + const base = filename.slice(0, -".script.json".length); + // A dbt script's modules ARE its dbt project, so it keeps the flat + // layout: only the project goes in the folder, which is what + // `--project-dir` expects and what makes the import a plain copy. + if (!isDbtModules(parsed.modules)) { + _moduleScriptPaths.add(base); + } } } catch {} } @@ -926,7 +1044,7 @@ function ZipFSElement( let finalPath = transformPath(); // Redirect content files for scripts with modules into __mod/ folder - if (kind == "other" && exts.some((ext) => p.endsWith(ext))) { + if (kind == "other" && hasScriptExt(p)) { const normalizedP = p.replace(/^\.[\\/]/, ""); const moduleScripts = await getModuleScriptPaths(); for (const basePath of moduleScripts) { @@ -934,7 +1052,11 @@ function ZipFSElement( const ext = normalizedP.slice(basePath.length); // e.g., ".ts", ".py" const dir = path.dirname(finalPath); const base = path.basename(basePath); - finalPath = path.join(dir, base + getModuleFolderSuffix(), "script" + ext); + finalPath = path.join( + dir, + base + getModuleFolderSuffix(), + "script" + ext, + ); break; } } @@ -955,7 +1077,9 @@ function ZipFSElement( } let inlineScripts; try { - const assigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }); + const assigner = newPathAssigner(defaultTs, { + skipInlineScriptSuffix: getNonDottedPaths(), + }); // Preserve original !inline filenames from the flow to avoid phantom renames const inlineMapping = extractCurrentMapping( flow.value.modules as any, @@ -969,27 +1093,40 @@ function ZipFSElement( SEP, defaultTs, assigner, - { skipInlineScriptSuffix: getNonDottedPaths(), failOnInlineDirective: true }, + { + skipInlineScriptSuffix: getNonDottedPaths(), + failOnInlineDirective: true, + }, ); if (flow.value.failure_module) { - inlineScripts.push(...extractInlineScriptsForFlows( - [flow.value.failure_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { skipInlineScriptSuffix: getNonDottedPaths(), failOnInlineDirective: true }, - )); + inlineScripts.push( + ...extractInlineScriptsForFlows( + [flow.value.failure_module], + inlineMapping, + SEP, + defaultTs, + assigner, + { + skipInlineScriptSuffix: getNonDottedPaths(), + failOnInlineDirective: true, + }, + ), + ); } if (flow.value.preprocessor_module) { - inlineScripts.push(...extractInlineScriptsForFlows( - [flow.value.preprocessor_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { skipInlineScriptSuffix: getNonDottedPaths(), failOnInlineDirective: true }, - )); + inlineScripts.push( + ...extractInlineScriptsForFlows( + [flow.value.preprocessor_module], + inlineMapping, + SEP, + defaultTs, + assigner, + { + skipInlineScriptSuffix: getNonDottedPaths(), + failOnInlineDirective: true, + }, + ), + ); } } catch (error) { log.error( @@ -1038,7 +1175,9 @@ function ZipFSElement( inlineScripts = extractInlineScriptsForApps( undefined, app?.["value"], - newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }), + newPathAssigner(defaultTs, { + skipInlineScriptSuffix: getNonDottedPaths(), + }), (_, val) => val["name"], false, ); @@ -1124,7 +1263,7 @@ function ZipFSElement( isDirectory: false, path: path.join(finalPath, filePath.substring(1)), async *getChildren() {}, - async getContentText() { + async getContentText() { if (typeof content !== "string") { throw new Error( `Content of raw app file ${filePath} is not a string`, @@ -1213,7 +1352,9 @@ function ZipFSElement( // Simplify fields for cleaner YAML output if (simplifiedRunnable.fields) { - simplifiedRunnable.fields = simplifyFields(simplifiedRunnable.fields); + simplifiedRunnable.fields = simplifyFields( + simplifiedRunnable.fields, + ); } yield { @@ -1272,17 +1413,28 @@ function ZipFSElement( log.error(`Failed to parse script.yaml at path: ${p}`); throw error; } - const hasModules = parsed["modules"] && Object.keys(parsed["modules"]).length > 0; + const hasModules = + parsed["modules"] && Object.keys(parsed["modules"]).length > 0; + // A dbt script's module folder holds its dbt project and dbt's own + // files, so its lock stays outside like a plain script's — only the + // descriptor lives in there. + const isDbtScript = isDbtModules(parsed["modules"]); if ( parsed["lock"] && parsed["lock"] != "" && parsed["codebase"] == undefined ) { - if (hasModules) { + if (hasModules && !isDbtScript) { // Lock lives inside __mod/ folder as script.lock - const scriptBase = removeSuffix(removeSuffix(p.replaceAll(SEP, "/"), ".json"), ".script"); + const scriptBase = removeSuffix( + removeSuffix(p.replaceAll(SEP, "/"), ".json"), + ".script", + ); parsed["lock"] = - "!inline " + scriptBase + getModuleFolderSuffix() + "/script.lock"; + "!inline " + + scriptBase + + getModuleFolderSuffix() + + "/script.lock"; } else { parsed["lock"] = "!inline " + @@ -1322,8 +1474,7 @@ function ZipFSElement( throw error; } const resourceType = parsed["resource_type"]; - const formatExtension = - resourceTypeToFormatExtension[resourceType]; + const formatExtension = resourceTypeToFormatExtension[resourceType]; const isFileset = resourceTypeToIsFileset[resourceType] ?? false; if (isFileset) { @@ -1388,18 +1539,24 @@ function ZipFSElement( throw error; } const lock = parsed["lock"]; - const scriptModules: Record | undefined = parsed["modules"]; + const scriptModules: Record | undefined = + parsed["modules"]; const hasModules = scriptModules && Object.keys(scriptModules).length > 0; + // A dbt script's module folder is its dbt project, so the metadata and + // lock stay beside it — the descriptor is the one Windmill file that goes + // in, because it is the script's content. + const isDbt = isDbtModules(scriptModules); // Compute base path and module folder const metaExt = useYaml ? ".yaml" : ".json"; const scriptBasePath = removeSuffix( removeSuffix(finalPath, metaExt), - ".script" + ".script", ); - const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(); + const moduleFolderPath = + scriptBasePath + getModuleFolderSuffix(isDbt ? "dbt" : undefined); - if (hasModules) { + if (hasModules && !isDbt) { // Redirect metadata into __mod/script.yaml r[0].path = path.join(moduleFolderPath, "script" + metaExt); } @@ -1407,9 +1564,10 @@ function ZipFSElement( if (lock && lock != "") { r.push({ isDirectory: false, - path: hasModules - ? path.join(moduleFolderPath, "script.lock") - : removeSuffix(finalPath, metaExt) + ".lock", + path: + hasModules && !isDbt + ? path.join(moduleFolderPath, "script.lock") + : removeSuffix(finalPath, metaExt) + ".lock", async *getChildren() {}, async getContentText() { return lock; @@ -1436,7 +1594,7 @@ function ZipFSElement( // Yield the module lock file if present if (mod.lock) { - const baseName = relPath.replace(/\.[^.]+$/, ''); + const baseName = relPath.replace(/\.[^.]+$/, ""); yield { isDirectory: false, path: path.join(moduleFolderPath, baseName + ".lock"), @@ -1464,11 +1622,14 @@ function ZipFSElement( throw error; } const resourceType = parsed["resource_type"]; - const formatExtension = - resourceTypeToFormatExtension[resourceType]; + const formatExtension = resourceTypeToFormatExtension[resourceType]; const isFileset = resourceTypeToIsFileset[resourceType] ?? false; - if (isFileset && typeof parsed["value"] === "object" && parsed["value"] !== null) { + if ( + isFileset && + typeof parsed["value"] === "object" && + parsed["value"] !== null + ) { const filesetBasePath = removeSuffix(finalPath, ".resource.json") + ".fileset"; // Push directory entry for the fileset @@ -1476,7 +1637,9 @@ function ZipFSElement( isDirectory: true, path: filesetBasePath, async *getChildren() { - for (const [relPath, fileContent] of Object.entries(parsed["value"])) { + for (const [relPath, fileContent] of Object.entries( + parsed["value"], + )) { if (typeof fileContent === "string") { yield { isDirectory: false, @@ -1652,9 +1815,15 @@ export async function elementsToMap( } const path = entry.path; // Include module files in the map so they're compared for changes, - // but they're pushed as part of their parent script via handleFile + // but they're pushed as part of their parent script via handleFile. + // `--skip-scripts` therefore covers them, and has to be applied here: the + // filters below are past this shortcut, so a changed module would push the + // parent script the flag asked to leave alone — every file of a dbt project + // is one of these. if (isScriptModulePath(path)) { - map[path] = await entry.getContentText(); + if (!skips.skipScripts) { + map[path] = await entry.getContentText(); + } continue; } if ( @@ -1663,6 +1832,10 @@ export async function elementsToMap( !isRawAppFile(path) && !isWorkspaceDependencies(path) ) { + // The metadata format decides which of the two metadata twins is read, + // and drops the other. A dbt descriptor is not metadata and is not + // reached here: it lives inside the project folder, so the module branch + // above already took it, in both modes. if (json && path.endsWith(".yaml")) continue; if (!json && path.endsWith(".json")) continue; @@ -1695,9 +1868,16 @@ export async function elementsToMap( } if (isRawAppFile(path)) { - const suffix = path.split(getFolderSuffix("raw_app") + SEP).pop(); + // FSFSElement builds paths with the platform separator, while the checks + // below are written with "/": without normalizing, none of them match on + // Windows and the push collector's own exclusions become perpetual diffs. + const suffix = path + .split(getFolderSuffix("raw_app") + SEP) + .pop() + ?.replaceAll(SEP, "/"); if ( suffix?.startsWith("dist/") || + suffix?.startsWith(RECORDINGS_FOLDER + "/") || suffix == "wmill.d.ts" || suffix == "package-lock.json" || suffix == "DATATABLES.md" @@ -1706,7 +1886,11 @@ export async function elementsToMap( } } - if (skips.skipResources && (isFileResource(path) || isFilesetResource(path))) continue; + if ( + skips.skipResources && + (isFileResource(path) || isFilesetResource(path)) + ) + continue; const ext = json ? ".json" : ".yaml"; if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue; @@ -1749,6 +1933,8 @@ export async function elementsToMap( fileType === "workspace_dependencies" ) continue; + if (skips.skipDatatableMigrations && fileType === "datatable_migration") + continue; } catch { // If getTypeStrFromPath can't determine the type, continue processing the file } @@ -1827,17 +2013,38 @@ export async function elementsToMap( if (wrongFormatPaths.length > 0) { const isNonDotted = getNonDottedPaths(); - const foundFormat = isNonDotted ? ".flow/.app/.raw_app" : "__flow/__app/__raw_app"; - const expectedFormat = isNonDotted ? "__flow/__app/__raw_app" : ".flow/.app/.raw_app"; + const foundFormat = isNonDotted + ? ".flow/.app/.raw_app" + : "__flow/__app/__raw_app"; + const expectedFormat = isNonDotted + ? "__flow/__app/__raw_app" + : ".flow/.app/.raw_app"; const configHint = isNonDotted ? "Either remove 'nonDottedPaths: true' from wmill.yaml, or rename these directories to use __flow/__app/__raw_app format." : "Either add 'nonDottedPaths: true' to wmill.yaml, or rename these directories to use .flow/.app/.raw_app format."; const pathList = wrongFormatPaths.map((p) => ` ${p}`).join("\n"); throw new Error( - `Found ${wrongFormatPaths.length} directory(ies) using ${foundFormat} format, but wmill.yaml expects ${expectedFormat}:\n${pathList}\n${configHint}` + `Found ${wrongFormatPaths.length} directory(ies) using ${foundFormat} format, but wmill.yaml expects ${expectedFormat}:\n${pathList}\n${configHint}`, ); } + // A dbt project's descriptor is optional, and the two sides spell "absent" + // differently: nothing on disk, and nothing in the export (which omits an + // empty one so a project that never named a descriptor never grows one). + // Left alone that reads as an addition on every push and a deletion on every + // pull, forever. Both sides are given the empty descriptor the absence means, + // so a descriptor-less project reaches a clean sync state. + for (const key of Object.keys(map)) { + // Normalized first: the local map's keys are built with `path.join`, so on + // Windows this reads `__dbt\\dbt_project.yml` and an unnormalized match + // would synthesize nothing — leaving exactly the perpetual push/pull diff + // above unguarded, on that platform only. + if (!key.replaceAll("\\", "/").endsWith("__dbt/dbt_project.yml")) continue; + const descriptor = + key.slice(0, -"dbt_project.yml".length) + DBT_DESCRIPTOR_NAME; + if (!(descriptor in map)) map[descriptor] = ""; + } + return map; } @@ -1851,6 +2058,7 @@ export interface Skips { skipApps?: boolean | undefined; skipFolders?: boolean | undefined; skipWorkspaceDependencies?: boolean | undefined; + skipDatatableMigrations?: boolean | undefined; skipScriptsMetadata?: boolean | undefined; includeSchedules?: boolean | undefined; includeTriggers?: boolean | undefined; @@ -1967,7 +2175,11 @@ export function canonicalizeCaseInsensitiveKeys( const lk = seg.toLowerCase(); let entry = node.children.get(lk); if (!entry) { - entry = { canonical: seg, ambiguous: false, node: { children: new Map() } }; + entry = { + canonical: seg, + ambiguous: false, + node: { children: new Map() }, + }; node.children.set(lk, entry); } else if (entry.canonical !== seg) { entry.ambiguous = true; @@ -2141,7 +2353,9 @@ export function preservePendingScriptLocks( remoteParsed = isYaml ? yamlParseContent(metaKey, remote[metaKey]) : JSON.parse(remote[metaKey]); - localParsed = isYaml ? yamlParseContent(metaKey, localMeta) : JSON.parse(localMeta); + localParsed = isYaml + ? yamlParseContent(metaKey, localMeta) + : JSON.parse(localMeta); } catch { continue; } @@ -2155,7 +2369,8 @@ export function preservePendingScriptLocks( // The local side must reference an inline lock backed by a committed file. const localLock = localParsed["lock"]; - if (typeof localLock !== "string" || !localLock.startsWith("!inline ")) continue; + if (typeof localLock !== "string" || !localLock.startsWith("!inline ")) + continue; // Derive the lock-file key from the `!inline` reference itself, not from the // metadata path: a multi-module script keeps its lock at `…__mod/script.lock`, @@ -2188,10 +2403,37 @@ async function compareDynFSElement( ): Promise<{ changes: Change[]; localMap: Record }> { let [m1, m2] = els2 ? await Promise.all([ - elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), - elementsToMap(els2, ignore, json, skips, specificItems, branchOverride, !isEls1Remote), + elementsToMap( + els1, + ignore, + json, + skips, + specificItems, + branchOverride, + isEls1Remote, + ), + elementsToMap( + els2, + ignore, + json, + skips, + specificItems, + branchOverride, + !isEls1Remote, + ), ]) - : [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), {}]; + : [ + await elementsToMap( + els1, + ignore, + json, + skips, + specificItems, + branchOverride, + isEls1Remote, + ), + {}, + ]; // Reconcile letter-case differences between the local tree and the // authoritative server casing. Only meaningful for an actual two-sided diff @@ -2506,9 +2748,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { ); } - // Files inside __mod/ folders are script module files — always valid wmill files + // Files inside a module folder belong to their parent script, so they are + // always valid wmill files — except the ones dbt generates, which are not + // part of the bundle and must not surface as items of their own. if (isScriptModulePath(p)) { - return false; + return isDbtGeneratedPath(p); } try { @@ -2560,6 +2804,7 @@ export async function ignoreF(wmillconf: { extraIncludes?: string[]; skipResourceTypes?: boolean; skipWorkspaceDependencies?: boolean; + skipDatatableMigrations?: boolean; json?: boolean; includeUsers?: boolean; includeGroups?: boolean; @@ -2624,6 +2869,14 @@ export async function ignoreF(wmillconf: { ) { return false; // Don't ignore workspace dependencies (they are always included unless explicitly skipped) } + // `migrations/datatable/**` is outside the u/f/g namespaces the path + // filters are written against, so the skip flag is its only control. + if ( + !wmillconf.skipDatatableMigrations && + fileType === "datatable_migration" + ) { + return false; + } } catch { // If getTypeStrFromPath can't determine the type, fall through to normal logic } @@ -2637,6 +2890,46 @@ export async function ignoreF(wmillconf: { }; } +/** + * How many migration *records* a set of changed files covers. One migration is two + * files (`.up.sql` + optional `.down.sql`) for a single `(datatable, timestamp)`, + * so counting paths would overstate what a prompt is about to delete. + */ +export function countDatatableMigrationRecords( + changes: { path: string }[], +): number { + const records = new Set(); + for (const c of changes) { + const parsed = parseDatatableMigrationPath(c.path); + if (parsed) records.add(`${parsed.datatable}\0${parsed.timestamp}`); + } + return records.size; +} + +/** + * The `deleted` changes for data table migrations that a push cannot safely trust. + * + * Migrations bypass the repo's path filters (they live outside `f/`/`u/`), so a clone + * made before they were synced sees every server-side migration as remote-only — and + * `pushMigrationFromDisk` reads a locally absent `.up.sql` as an instruction to delete + * it. `recorded` is what this repository's own history says (see + * `gitRecordedDatatableMigrationPaths`): a recorded path was genuinely tracked, so its + * absence now is a real deletion; a path missing from a `known` set is one this branch + * has never had, and deleting it is a guess. `unknown` history is not evidence of + * anything, so nothing is trusted. The caller confirms whatever comes back explicitly + * and never deletes it unattended. + */ +export function untrackedDatatableMigrationDeletions< + T extends { name: string; path: string }, +>(changes: T[], recorded: RecordedMigrationPaths): T[] { + return changes.filter( + (c) => + c.name === "deleted" && + isDatatableMigrationPath(c.path) && + !(recorded.kind === "known" && recorded.paths.has(c.path)), + ); +} + interface ChangeTracker { scripts: string[]; flows: string[]; @@ -2644,13 +2937,78 @@ interface ChangeTracker { rawApps: string[]; } +/// The script a module file belongs to, added to the tracker so its top hash is +/// refreshed. Derived with `getScriptBasePathFromModulePath`, which normalizes +/// separators: searching the raw path for `__dbt/` found nothing on Windows, +/// where the folder is spelled `__dbt\`, so every model edit there was skipped. +async function addModuleParentToChanged(p: string, tracker: ChangeTracker) { + // A folder layout's METADATA — `__mod/script.yaml` — is an entry-point + // path too, and it is not a content file: pushed as one, the metadata pass + // asks `inferContentTypeFromFilePath` for the language of `.yaml` and aborts + // the whole command. It resolves to its content file like any other metadata. + if (isModuleEntryMetadata(p)) { + try { + const contentPath = await findContentFile(p); + if (contentPath && !tracker.scripts.includes(contentPath)) { + tracker.scripts.push(contentPath); + } + } catch { + // ignore — content file not found + } + return; + } + if (isModuleEntryPoint(p)) { + // Entry point (e.g. __mod/script.ts) IS the parent script content file. + if (!tracker.scripts.includes(p)) { + tracker.scripts.push(p); + } + return; + } + const scriptBasePath = getScriptBasePathFromModulePath(p); + if (scriptBasePath === undefined) { + return; + } + const push = async (candidate: string): Promise => { + try { + const contentPath = await findContentFile(candidate); + if (contentPath && !tracker.scripts.includes(contentPath)) { + tracker.scripts.push(contentPath); + } + return contentPath != undefined; + } catch { + return false; + } + }; + // A dbt script's metadata sits beside its folder; the descriptor is inside. + if (isDbtModulePath(p)) { + await push(scriptBasePath + ".script.yaml"); + return; + } + // Folder layout first (`__mod/script.{ext}`), then flat. + if (!(await push(scriptBasePath + getModuleFolderSuffix() + "/script.yaml"))) { + await push(scriptBasePath + ".script.yaml"); + } +} + async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { // Datatable migration .sql files are not scripts; they're synced via the // dedicated datatable_migration handler in the push loop. if (isDatatableMigrationPath(p)) { return; } - const isScript = exts.some((e) => p.endsWith(e)) && !isFileResource(p) && !isFilesetResource(p); + // Module files first, and whatever their extension: a dbt project authors + // `dbt_project.yml`, `packages.yml`, schema YAML and seed CSVs, none of which + // are Windmill script extensions — gated behind that test they never reached + // the tracker, so the top hash in `wmill-lock.yaml` (which covers the modules) + // stayed stale for exactly the files a dbt project is mostly made of. + if (isScriptModulePath(p)) { + await addModuleParentToChanged(p, tracker); + return; + } + const isScript = + hasScriptExt(p) && + !isFileResource(p) && + !isFilesetResource(p); if (isScript) { if (isFlowPath(p)) { const folder = extractFolderPath(p, "flow")!; @@ -2667,37 +3025,6 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { if (!tracker.rawApps.includes(folder)) { tracker.rawApps.push(folder); } - } else if (isScriptModulePath(p)) { - if (isModuleEntryPoint(p)) { - // Entry point (e.g. __mod/script.ts) IS the parent script content file - if (!tracker.scripts.includes(p)) { - tracker.scripts.push(p); - } - } else { - // Module file changed — find the parent script content file - const moduleSuffix = getModuleFolderSuffix() + "/"; - const idx = p.indexOf(moduleSuffix); - if (idx !== -1) { - const scriptBasePath = p.substring(0, idx); - // Try folder layout first: __mod/script.{ext} - try { - const contentPath = await findContentFile(scriptBasePath + getModuleFolderSuffix() + "/script.yaml"); - if (contentPath && !tracker.scripts.includes(contentPath)) { - tracker.scripts.push(contentPath); - } - } catch { - // Fall back to flat layout: scriptBasePath.script.yaml - try { - const contentPath = await findContentFile(scriptBasePath + ".script.yaml"); - if (contentPath && !tracker.scripts.includes(contentPath)) { - tracker.scripts.push(contentPath); - } - } catch { - // ignore — content file not found - } - } - } - } } else { if (!tracker.scripts.includes(p)) { tracker.scripts.push(p); @@ -2715,7 +3042,7 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { } } -async function buildTracker(changes: Change[]) { +export async function buildTracker(changes: Change[]) { const tracker: ChangeTracker = { scripts: [], flows: [], @@ -2734,7 +3061,7 @@ async function buildTracker(changes: Change[]) { * When a module file changes, find and push the parent script. * The parent script's handleFile will read the __mod/ folder and include all modules. */ -async function pushParentScriptForModule( +export async function pushParentScriptForModule( modulePath: string, workspace: Workspace, alreadySynced: string[], @@ -2743,11 +3070,82 @@ async function pushParentScriptForModule( rawWorkspaceDependencies: Record, codebases: SyncCodebase[], ): Promise { - const moduleSuffix = getModuleFolderSuffix() + "/"; - const idx = modulePath.indexOf(moduleSuffix); - if (idx === -1) return; - const scriptBasePath = modulePath.substring(0, idx); - const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(); + const isDbt = isDbtModulePath(modulePath); + // Via the shared helper, which normalizes separators: a Windows path spells + // the folder `__dbt\\`, and searching the raw path for `__dbt/` would find + // nothing and silently return without deploying the parent — while the caller + // still records the file as synced. + const scriptBasePath = getScriptBasePathFromModulePath(modulePath); + if (scriptBasePath === undefined) return; + const moduleFolderPath = + scriptBasePath + getModuleFolderSuffix(isDbt ? "dbt" : undefined); + + // A dbt project's descriptor sits INSIDE its folder (`__dbt/wm_dbt.yaml`) and + // is optional, so the project itself is what identifies the script. + if (isDbt) { + // Only the LOOKUP is tolerated: a module under no script's project is a + // stray file, not an error. Deploying it is not — swallowing that would let + // a module-only push report success while the remote project is unchanged. + // BEFORE the lookup, because the lookup succeeds whenever a descriptor is + // there: `dbt_project.yml` is what makes the bundle a project, and pushing + // without it replaces a healthy deployment with one whose dependency job + // fails for having no project at all. + const hasMetadata = + existsSync(scriptBasePath + ".script.yaml") || + existsSync(scriptBasePath + ".script.json"); + if (!existsSync(moduleFolderPath + "/dbt_project.yml")) { + if (hasMetadata) { + throw new Error( + `${moduleFolderPath} has no dbt_project.yml but ${scriptBasePath}.script.yaml ` + + `remains, so there is no dbt project left to push. Delete the metadata too to ` + + `archive the script, or restore the project.` + ); + } + // Nothing local claims this script any more — neither project nor + // metadata — so it is archived like any other locally deleted item. The + // deletions arrive one file at a time, hence `alreadySynced`. + const remote = scriptBasePath.replaceAll(SEP, "/"); + if (!alreadySynced.includes(remote)) { + alreadySynced.push(remote); + log.info(`Archiving script ${remote}`); + await wmill + .archiveScriptByPath({ workspace: workspace.workspaceId, path: remote }) + .catch((e: any) => { + // Only "already gone" is the state we wanted. An auth, network or + // server failure must fail the push: swallowing it reports success + // while the project stays deployed, which is the thing this branch + // exists to prevent. + if (e?.status !== 404) throw e; + log.debug(`${remote} was already gone remotely`); + }); + } + return; + } + let contentPath: string | undefined; + try { + contentPath = await findContentFile(scriptBasePath + ".script.yaml"); + } catch (e) { + // A path claimed by two scripts is not a parent that cannot be found: + // swallowed here, `wmill sync push` reports success on a model edit that + // deployed nothing, and the collision stays invisible until the ordinary + // script overwrites the project. + if (e instanceof DbtPathCollisionError) throw e; + log.debug(`Could not find parent script for dbt module: ${modulePath}`); + return; + } + if (contentPath) { + await handleFile( + contentPath, + workspace, + alreadySynced, + message, + opts, + rawWorkspaceDependencies, + codebases, + ); + } + return; + } // Try folder layout first: look for script.{ext} inside __mod/ try { @@ -2958,11 +3356,16 @@ export async function pull( let specificItems = getSpecificItemsForCurrentBranch(opts, wsNameForConfig); // Compute the workspace name for file naming (default to workspaceId) - let wsNameForFiles = wsNameForConfig ? resolveWsNameForFiles(opts, wsNameForConfig) : workspace.workspaceId; + let wsNameForFiles = wsNameForConfig + ? resolveWsNameForFiles(opts, wsNameForConfig) + : workspace.workspaceId; // Augment specificItems with server-side ws_specific entries const localSpecificItems = specificItems; - const wsSpecificMerge = await mergeWsSpecificFromServer(workspace.workspaceId, specificItems); + const wsSpecificMerge = await mergeWsSpecificFromServer( + workspace.workspaceId, + specificItems, + ); specificItems = wsSpecificMerge.merged; // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) @@ -3003,7 +3406,9 @@ export async function pull( opts.includeSettings, opts.includeKey, opts.skipWorkspaceDependencies, + opts.skipDatatableMigrations, opts.defaultTs, + opts.syncBehavior, ); const remote = ZipFSElement( @@ -3108,7 +3513,8 @@ export async function pull( const conflicts = []; log.info(colors.gray(`Applying changes to files ...`)); - for await (const change of changes) { + for await (const rawChange of changes) { + const change: Change = rawChange; // Determine if this file should be written to a workspace-specific path let targetPath = change.path; if (specificItems && isSpecificItem(change.path, specificItems)) { @@ -3124,6 +3530,28 @@ export async function pull( const target = path.join(process.cwd(), targetPath); const stateTarget = path.join(process.cwd(), ".wmill", targetPath); + // An empty dbt descriptor is not a file: the remote spells "this project + // named no descriptor" as empty content, and writing that would put a + // Windmill file inside a project that has none. ABSENCE is the state to + // reach, so both copies are removed if present and their being missing — + // a project pulled for the first time — is the goal, not an error. The + // `.wmill` copy goes too, or the same change is reported on every pull. + // + // `force` covers the missing file and NOTHING else: a permission or + // read-only-filesystem failure has to surface, or the pull reports + // success while the old descriptor — its warehouse, its command, its + // arguments — is still what runs locally. + if ( + isDbtDescriptorPath(change.path) && + ((change.name === "added" && change.content === "") || + (change.name === "edited" && change.after === "")) + ) { + await rm(target, { force: true }); + if (opts.stateful) { + await rm(stateTarget, { force: true }); + } + continue; + } if (change.name === "edited") { if (opts.stateful) { try { @@ -3165,11 +3593,13 @@ export async function pull( // ignore } } - if (exts.some((e) => change.path.endsWith(e))) { + if (hasScriptExt(change.path)) { log.info( `Editing script content of ${targetPath}${ targetPath !== change.path - ? colors.gray(` (workspace-specific override for ${change.path})`) + ? colors.gray( + ` (workspace-specific override for ${change.path})`, + ) : "" }`, ); @@ -3180,7 +3610,9 @@ export async function pull( log.info( `Editing ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path - ? colors.gray(` (workspace-specific override for ${change.path})`) + ? colors.gray( + ` (workspace-specific override for ${change.path})`, + ) : "" }`, ); @@ -3198,7 +3630,9 @@ export async function pull( log.info( `Adding ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path - ? colors.gray(` (workspace-specific override for ${change.path})`) + ? colors.gray( + ` (workspace-specific override for ${change.path})`, + ) : "" }`, ); @@ -3215,18 +3649,15 @@ export async function pull( await copyFile(target, stateTarget); } } else if (change.name === "deleted") { - try { - log.info( - `Deleting ${changeTypeLabel(change.path)}${change.path}`, - ); - await rm(target); - if (opts.stateful) { - await rm(stateTarget); - } - } catch { - if (opts.stateful) { - await rm(stateTarget); - } + log.info(`Deleting ${changeTypeLabel(change.path)}${change.path}`); + // `force` on both: the goal is that neither copy exists, and a file + // already absent — a dbt project's optional descriptor is never written + // — is that goal, not an error. Anything else (permissions, a read-only + // mount) surfaces rather than leaving a file the sync believes is gone. + // The state copy goes too, or the same deletion replays on every sync. + await rm(target, { force: true }); + if (opts.stateful) { + await rm(stateTarget, { force: true }); } } } @@ -3362,7 +3793,8 @@ export async function pull( try { // Dynamic import to avoid a circular dep between sync.ts and // generate-metadata.ts. Don't "clean up" to a static import. - const { rehashOnly } = await import("../generate-metadata/generate-metadata.ts"); + const { rehashOnly } = + await import("../generate-metadata/generate-metadata.ts"); // Reuse the local-side file list from the change-tracker so we don't // re-walk the filesystem. Apply the just-applied changes to derive the // post-pull state: localMap is pre-pull, but auto-fill needs to see @@ -3381,7 +3813,7 @@ export async function pull( log.info( colors.gray( `Auto-filled ${total} missing lockfile entr${total === 1 ? "y" : "ies"} ` + - `(${filled.scripts} script, ${filled.flows} flow, ${filled.apps} app) from disk.`, + `(${filled.scripts} script, ${filled.flows} flow, ${filled.apps} app) from disk.`, ), ); } @@ -3469,17 +3901,14 @@ export async function gitDeploy( // the wm_deploy branch). Mirrors the hub script's `--promotion `. const promotion = useIndividualBranch && !opts.promotion - ? getCurrentGitBranch() ?? undefined + ? (getCurrentGitBranch() ?? undefined) : opts.promotion; await pull({ ...opts, yes: true, skipBranchValidation: true, - extraIncludes: [ - ...(opts.extraIncludes ?? []), - ...includes.extraIncludes, - ], + extraIncludes: [...(opts.extraIncludes ?? []), ...includes.extraIncludes], // Workspace-wide mode force-includes the deployed default-excluded kinds // (full mirror). Individual-branch/promotion mode forces nothing — these // keys stay ABSENT so pull resolves them from the promotion target's @@ -3527,7 +3956,9 @@ function prettyChanges( const folderNote = folderDefaultAnnotations?.get(change.path); const extraNote = folderNote - ? colors.cyan(` (will be permissioned as ${folderNote} via folder default)`) + ? colors.cyan( + ` (will be permissioned as ${folderNote} via folder default)`, + ) : ""; if (change.name === "added") { @@ -3672,14 +4103,12 @@ async function checkServerLockJobs( const pending = (queued as { script_path?: string }[]).filter((j) => belongsToPush(j.script_path), ).length; - const failed = ( - completed as { script_path?: string; result?: unknown }[] - ) + const failed = (completed as { script_path?: string; result?: unknown }[]) .filter((j) => belongsToPush(j.script_path)) .map((j) => ({ path: j.script_path!, - error: (j.result as { error?: { message?: string } } | undefined) - ?.error?.message, + error: (j.result as { error?: { message?: string } } | undefined)?.error + ?.message, })); return { pending, failed }; } catch { @@ -3689,7 +4118,12 @@ async function checkServerLockJobs( } export async function push( - opts: GlobalOptions & SyncOptions & { repository?: string; branch?: string; acceptOverridingPermissionedAsWithSelf?: boolean }, + opts: GlobalOptions & + SyncOptions & { + repository?: string; + branch?: string; + acceptOverridingPermissionedAsWithSelf?: boolean; + }, ) { if ((opts as any).jsonOutput) log.setSilent(true); // Save original CLI options before merging with config file @@ -3752,7 +4186,9 @@ export async function push( let specificItems = getSpecificItemsForCurrentBranch(opts, wsNameForConfig); // Compute the workspace name for file naming (default to workspaceId) - let wsNameForFiles = wsNameForConfig ? resolveWsNameForFiles(opts, wsNameForConfig) : workspace.workspaceId; + let wsNameForFiles = wsNameForConfig + ? resolveWsNameForFiles(opts, wsNameForConfig) + : workspace.workspaceId; // Keep the pre-merge specificItems so we can detect entries that are // flagged locally but not yet ws_specific on the server (post-merge would @@ -3760,7 +4196,10 @@ export async function push( const localSpecificItems = specificItems; // Augment specificItems with server-side ws_specific entries - const wsSpecificMerge = await mergeWsSpecificFromServer(workspace.workspaceId, specificItems); + const wsSpecificMerge = await mergeWsSpecificFromServer( + workspace.workspaceId, + specificItems, + ); specificItems = wsSpecificMerge.merged; const serverWsSpecificItems = wsSpecificMerge.serverItems; @@ -3844,7 +4283,9 @@ export async function push( opts.includeSettings, opts.includeKey, opts.skipWorkspaceDependencies, + opts.skipDatatableMigrations, opts.defaultTs, + opts.syncBehavior, ))!, !opts.json, opts.defaultTs ?? "bun", @@ -3854,7 +4295,11 @@ export async function push( parseSyncBehavior(opts.syncBehavior) >= 1, ); - const local = await FSFSElement(path.join(process.cwd(), ""), codebases, false); + const local = await FSFSElement( + path.join(process.cwd(), ""), + codebases, + false, + ); const { changes, localMap } = await compareDynFSElement( local, remote, @@ -4120,14 +4565,20 @@ export async function push( let triggerCount = 0; for await (const entry of readDirRecursiveWithIgnore(() => false, local)) { if (entry.isDirectory) continue; - if (!opts.includeSchedules && entry.path.endsWith(".schedule.yaml")) scheduleCount++; - if (!opts.includeTriggers && entry.path.endsWith("_trigger.yaml")) triggerCount++; + if (!opts.includeSchedules && entry.path.endsWith(".schedule.yaml")) + scheduleCount++; + if (!opts.includeTriggers && entry.path.endsWith("_trigger.yaml")) + triggerCount++; } if (scheduleCount > 0) { - skippedWarnings.push(`Skipping ${scheduleCount} schedule file(s). Use --include-schedules or set includeSchedules: true in wmill.yaml`); + skippedWarnings.push( + `Skipping ${scheduleCount} schedule file(s). Use --include-schedules or set includeSchedules: true in wmill.yaml`, + ); } if (triggerCount > 0) { - skippedWarnings.push(`Skipping ${triggerCount} trigger file(s). Use --include-triggers or set includeTriggers: true in wmill.yaml`); + skippedWarnings.push( + `Skipping ${triggerCount} trigger file(s). Use --include-triggers or set includeTriggers: true in wmill.yaml`, + ); } for (const warning of skippedWarnings) { log.warn(warning); @@ -4137,6 +4588,44 @@ export async function push( await fetchRemoteVersion(workspace); + const recordedMigrationPaths: RecordedMigrationPaths = changes.some( + (c) => c.name === "deleted" && isDatatableMigrationPath(c.path), + ) + ? gitRecordedDatatableMigrationPaths() + : { kind: "known", paths: new Set() }; + const ambiguousMigrationDeletions = untrackedDatatableMigrationDeletions( + changes, + recordedMigrationPaths, + ); + const keepAmbiguousMigrationsOnRemote = () => { + log.info( + colors.yellow( + `Keeping ${countDatatableMigrationRecords(ambiguousMigrationDeletions)} data table migration(s) on the remote: ` + + (recordedMigrationPaths.kind === "known" + ? `this branch has never tracked them. Run 'wmill sync pull' to track them in git, or delete them from the workspace.` + : `${recordedMigrationPaths.reason}, so whether it ever tracked them cannot be established. ` + + `${recordedMigrationPaths.remedy} so a real deletion can be told apart, or delete them from the workspace.`), + ), + ); + const kept = changes.filter( + (c) => !ambiguousMigrationDeletions.includes(c), + ); + changes.length = 0; + changes.push(...kept); + }; + // An unattended run never resolves this ambiguity destructively, and a dry-run + // preview has to show what a push would really do — settle both before the + // change list is printed or serialized. A TTY push asks instead, after the + // user has seen the list. + let ambiguousMigrationsResolved = false; + if ( + ambiguousMigrationDeletions.length > 0 && + (opts.dryRun || opts.yes || !process.stdin.isTTY) + ) { + keepAmbiguousMigrationsOnRemote(); + ambiguousMigrationsResolved = true; + } + // Shared UI (the ui/ folder) is pushed out-of-band via pushSharedUi on apply // and is excluded from the file diff (isNotWmillFile), so surface its diff in // the dry-run preview. Without this the "Pull from repo" preview reads "no @@ -4218,7 +4707,18 @@ export async function push( `Run 'wmill folder add-missing' to create them locally, then push again.`; if (!userIsAdmin) { if (opts.jsonOutput) { - console.log(JSON.stringify({ success: false, error: "missing_folders", missing_folders: missingFolders, message: msg }, null, 2)); + console.log( + JSON.stringify( + { + success: false, + error: "missing_folders", + missing_folders: missingFolders, + message: msg, + }, + null, + 2, + ), + ); } else { log.error(msg); } @@ -4251,9 +4751,7 @@ export async function push( : {}), })), total: changes.length, - ...(changes.length > 0 - ? { warning: SYNC_PUSH_DESTRUCTIVE_WARNING } - : {}), + ...(changes.length > 0 ? { warning: SYNC_PUSH_DESTRUCTIVE_WARNING } : {}), }; console.log(JSON.stringify(result, null, 2)); return; @@ -4264,7 +4762,10 @@ export async function push( let folderDefaultAnnotations: Map | undefined; if (parseSyncBehavior(opts.syncBehavior) >= 1) { folderDefaultAnnotations = new Map(); - const folderRulesCache = new Map>(); + const folderRulesCache = new Map< + string, + Array<{ path_glob: string; permissioned_as: string }> + >(); for (const change of changes) { if (change.name !== "added") continue; const match = change.path.match(/^f\/([^/]+)\//); @@ -4272,14 +4773,26 @@ export async function push( const folderName = match[1]; if (!folderRulesCache.has(folderName)) { try { - const folder = await wmill.getFolder({ workspace: workspace.workspaceId, name: folderName }); - folderRulesCache.set(folderName, (folder as any).default_permissioned_as ?? []); + const folder = await wmill.getFolder({ + workspace: workspace.workspaceId, + name: folderName, + }); + folderRulesCache.set( + folderName, + (folder as any).default_permissioned_as ?? [], + ); } catch { folderRulesCache.set(folderName, []); } } const rules = folderRulesCache.get(folderName)!; - const remotePath = change.path.replace(/\.(script|schedule|http_trigger|websocket_trigger|kafka_trigger|nats_trigger|postgres_trigger|mqtt_trigger|amqp_trigger|sqs_trigger|gcp_trigger|azure_trigger|email_trigger)\.(yaml|json)$/, "").replace(/(\.flow|__flow)\/flow\.(yaml|json)$/, "").replace(/\.(app|raw_app)(\/app\.(yaml|json))?$/, ""); + const remotePath = change.path + .replace( + /\.(script|schedule|http_trigger|websocket_trigger|kafka_trigger|nats_trigger|postgres_trigger|mqtt_trigger|amqp_trigger|sqs_trigger|gcp_trigger|azure_trigger|email_trigger)\.(yaml|json)$/, + "", + ) + .replace(/(\.flow|__flow)\/flow\.(yaml|json)$/, "") + .replace(/\.(app|raw_app)(\/app\.(yaml|json))?$/, ""); const relative = remotePath.slice(`f/${folderName}/`.length); if (!relative) continue; for (const rule of rules) { @@ -4292,7 +4805,12 @@ export async function push( } if (!opts.jsonOutput) { - prettyChanges(changes, specificItems, wsNameForFiles, folderDefaultAnnotations); + prettyChanges( + changes, + specificItems, + wsNameForFiles, + folderDefaultAnnotations, + ); } if (opts.dryRun) { @@ -4306,7 +4824,9 @@ export async function push( const user = await wmill.whoami({ workspace: workspace.workspaceId }); const userIsAdminOrDeployer = user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug(`permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + ); permissionedAsContext = { userCache: new Map(), userIsAdminOrDeployer, @@ -4324,10 +4844,12 @@ export async function push( !!process.stdin.isTTY, ); } else if (folderDefaultAnnotations && folderDefaultAnnotations.size > 0) { - log.warn(colors.yellow( - `This workspace has folder default_permissioned_as rules that affect ${folderDefaultAnnotations.size} item(s) being pushed, ` + - `but syncBehavior is not set in wmill.yaml. Add 'syncBehavior: v1' to enable ownership preservation on update and on_behalf_of stripping on pull.` - )); + log.warn( + colors.yellow( + `This workspace has folder default_permissioned_as rules that affect ${folderDefaultAnnotations.size} item(s) being pushed, ` + + `but syncBehavior is not set in wmill.yaml. Add 'syncBehavior: v1' to enable ownership preservation on update and on_behalf_of stripping on pull.`, + ), + ); } // Reject malformed datatable migrations (duplicate timestamps, orphan downs) @@ -4358,6 +4880,18 @@ export async function push( return; } + if (ambiguousMigrationDeletions.length > 0 && !ambiguousMigrationsResolved) { + const deleteThem = await Confirm.prompt({ + message: + `Nothing in this repository's history accounts for ${countDatatableMigrationRecords(ambiguousMigrationDeletions)} migration definition(s), so it may simply never have synced them. ` + + `Delete them from the workspace anyway?`, + default: false, + }); + if (!deleteThem) { + keepAmbiguousMigrationsOnRemote(); + } + } + const start = performance.now(); const pushStartedAt = new Date().toISOString(); log.info(colors.gray(`Applying changes to files ...`)); @@ -4374,7 +4908,14 @@ export async function push( // Group changes by base path (before first dot) const groupedChanges = new Map(); for (const change of changes) { - const basePath = change.path.split(".")[0]; + // A module file is pushed by pushing its parent script, so it belongs in + // that script's group. Left in a group of its own it gets its own + // `alreadySynced`, and a push touching several files of one bundle then + // deploys the script once per file: several versions in a row, of which + // only the last is the one the asset graph ends up describing. + const basePath = + getScriptBasePathFromModulePath(change.path) ?? + change.path.split(".")[0]; if (!groupedChanges.has(basePath)) { groupedChanges.set(basePath, []); } @@ -4422,7 +4963,8 @@ export async function push( const effectiveParallelism = () => folderPhaseRemaining > 0 ? 1 : parallelizationFactor; // Cache git branch at the start to avoid repeated execSync calls per change - const cachedWsNameForPush = wsNameForFiles || (isGitRepository() ? getCurrentGitBranch() : null); + const cachedWsNameForPush = + wsNameForFiles || (isGitRepository() ? getCurrentGitBranch() : null); // Datatable migrations are two files (.up.sql/.down.sql) for one record, so // dedupe upsert/delete by (datatable, version) across the whole push. @@ -4562,13 +5104,19 @@ export async function push( const currentBranch = cachedWsNameForPush; let isFileResWsSpecific = false; - if (currentBranch && isWorkspaceSpecificFile(resourceFilePath)) { + if ( + currentBranch && + isWorkspaceSpecificFile(resourceFilePath) + ) { serverPath = fromWorkspaceSpecificPath( resourceFilePath, currentBranch, ); isFileResWsSpecific = true; - } else if (specificItems && isSpecificItem(change.path, specificItems)) { + } else if ( + specificItems && + isSpecificItem(change.path, specificItems) + ) { isFileResWsSpecific = true; } @@ -4612,7 +5160,8 @@ export async function push( // Check if this is a branch-specific item and get the original workspace-specific path let originalWorkspaceSpecificPath: string | undefined; - const isWsSpecific = specificItems && isSpecificItem(change.path, specificItems); + const isWsSpecific = + specificItems && isSpecificItem(change.path, specificItems); if (isWsSpecific) { originalWorkspaceSpecificPath = getWorkspaceSpecificPath( change.path, @@ -4628,13 +5177,17 @@ export async function push( newObj, opts.plainSecrets ?? false, alreadySynced, - opts.message, - originalWorkspaceSpecificPath, - permissionedAsContext, - isWsSpecific ? true : undefined, { - noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, - skipReencrypt: opts.skipReencryptOnKeyChange, + message: opts.message, + originalLocalPath: originalWorkspaceSpecificPath, + permissionedAsContext, + wsSpecific: isWsSpecific ? true : undefined, + keyPushOpts: { + noninteractive: + (opts.yes ?? false) || !process.stdin.isTTY, + skipReencrypt: opts.skipReencryptOnKeyChange, + }, + defaultTs: opts.defaultTs, }, ); @@ -4695,7 +5248,11 @@ export async function push( !isRawAppFile(change.path) && (change.path.endsWith(".script.json") || change.path.endsWith(".script.yaml") || - change.path.endsWith(".lock") || + // A `.lock` is the script's generated lockfile — except inside + // a dbt bundle, where the project may author one (`uv.lock`). + // Skipping it there would report the add on every push and + // never apply it, because no state file is written either. + (change.path.endsWith(".lock") && !isDbtModulePath(change.path)) || isFileResource(change.path)) ) { continue; @@ -4735,7 +5292,8 @@ export async function push( // Determine the actual local file path for this change // For branch-specific items, we read from workspace-specific files but push to base server paths let localFilePath = change.path; - const isAddedWsSpecific = specificItems && isSpecificItem(change.path, specificItems); + const isAddedWsSpecific = + specificItems && isSpecificItem(change.path, specificItems); if (isAddedWsSpecific) { const workspaceSpecificPath = getWorkspaceSpecificPath( change.path, @@ -4754,13 +5312,17 @@ export async function push( obj, opts.plainSecrets ?? false, [], - opts.message, - localFilePath, // Pass the actual local file path - permissionedAsContext, - isAddedWsSpecific ? true : undefined, { - noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, - skipReencrypt: opts.skipReencryptOnKeyChange, + message: opts.message, + originalLocalPath: localFilePath, + permissionedAsContext, + wsSpecific: isAddedWsSpecific ? true : undefined, + keyPushOpts: { + noninteractive: + (opts.yes ?? false) || !process.stdin.isTTY, + skipReencrypt: opts.skipReencryptOnKeyChange, + }, + defaultTs: opts.defaultTs, }, ); @@ -4768,7 +5330,9 @@ export async function push( await writeFile(stateTarget, change.content, "utf-8"); } } else if (change.name === "deleted") { - if (change.path.endsWith(".lock")) { + // Same as the added branch: a dbt project's own `.lock` is one of + // its files, so deleting it has to reach the parent script. + if (change.path.endsWith(".lock") && !isDbtModulePath(change.path)) { continue; } if (isScriptModulePath(change.path)) { @@ -4828,8 +5392,13 @@ export async function push( path: resourcePath, }); } catch (e: any) { - if (e?.status === 404 && deletedVarsResPaths.includes(resourcePath)) { - log.debug(`Resource ${resourcePath} already deleted by linked variable`); + if ( + e?.status === 404 && + deletedVarsResPaths.includes(resourcePath) + ) { + log.debug( + `Resource ${resourcePath} already deleted by linked variable`, + ); } else { throw e; } @@ -4848,7 +5417,10 @@ export async function push( // Metadata file deleted — delete the entire flow await wmill.deleteFlowByPath({ workspace: workspaceId, - path: removeSuffix(target, getDeleteSuffix("flow", "json")), + path: removeSuffix( + target, + getDeleteSuffix("flow", "json"), + ), }); } else { // Inline script file deleted within flow folder @@ -4871,7 +5443,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - opts.message, + { message: opts.message }, ); } else { // Flow folder doesn't exist locally — delete on server @@ -4890,7 +5462,10 @@ export async function push( // Metadata file deleted — delete the entire app await wmill.deleteApp({ workspace: workspaceId, - path: removeSuffix(target, getDeleteSuffix("app", "json")), + path: removeSuffix( + target, + getDeleteSuffix("app", "json"), + ), }); } else { // Inline script file deleted within app folder @@ -4913,7 +5488,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - opts.message, + { message: opts.message }, ); } else { // App folder doesn't exist locally — delete on server @@ -4932,7 +5507,10 @@ export async function push( // Delete the entire raw app await wmill.deleteApp({ workspace: workspaceId, - path: removeSuffix(target, getDeleteSuffix("raw_app", "json")), + path: removeSuffix( + target, + getDeleteSuffix("raw_app", "json"), + ), }); } else { const rawAppFolder = extractFolderPath(target, "raw_app"); @@ -4956,7 +5534,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - opts.message, + { message: opts.message, defaultTs: opts.defaultTs }, ); } else { // The entire raw app folder was deleted locally, @@ -5047,7 +5625,7 @@ export async function push( const triggerInfo = extractNativeTriggerInfo(change.path); if (!triggerInfo) { throw new Error( - `Invalid native trigger path: ${change.path}` + `Invalid native trigger path: ${change.path}`, ); } await wmill.deleteNativeTrigger({ @@ -5065,8 +5643,13 @@ export async function push( path: variablePath, }); } catch (e: any) { - if (e?.status === 404 && deletedVarsResPaths.includes(variablePath)) { - log.debug(`Variable ${variablePath} already deleted by linked resource`); + if ( + e?.status === 404 && + deletedVarsResPaths.includes(variablePath) + ) { + log.debug( + `Variable ${variablePath} already deleted by linked resource`, + ); } else { throw e; } @@ -5177,10 +5760,14 @@ export async function push( log.warn(`Failed to push shared UI folder: ${e}`); } try { - await offerToRunNewMigrations(workspace.workspaceId, newDatatableMigrations, { - yes: opts.yes, - jsonOutput: opts.jsonOutput, - }); + await offerToRunNewMigrations( + workspace.workspaceId, + newDatatableMigrations, + { + yes: opts.yes, + jsonOutput: opts.jsonOutput, + }, + ); } catch (e: any) { log.warn( `Failed to run new datatable migrations: ${e?.body ?? e?.message ?? e}`, @@ -5314,7 +5901,10 @@ const command = new Command() .option("--json", "Use JSON instead of YAML") .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") - .option("--include-secrets", "Include secrets in sync (overrides skipSecrets in wmill.yaml)") + .option( + "--include-secrets", + "Include secrets in sync (overrides skipSecrets in wmill.yaml)", + ) .option("--skip-resources", "Skip syncing resources") .option("--skip-resource-types", "Skip syncing resource types") .option("--skip-scripts", "Skip syncing scripts") @@ -5370,7 +5960,10 @@ const command = new Command() .option("--json", "Use JSON instead of YAML") .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") - .option("--include-secrets", "Include secrets in sync (overrides skipSecrets in wmill.yaml)") + .option( + "--include-secrets", + "Include secrets in sync (overrides skipSecrets in wmill.yaml)", + ) .option("--skip-resources", "Skip syncing resources") .option("--skip-resource-types", "Skip syncing resource types") .option("--skip-scripts", "Skip syncing scripts") @@ -5424,7 +6017,10 @@ const command = new Command() "--locks-required", "Fail if scripts or flow inline scripts that need locks have no locks", ) - .option("--auto-metadata", "Automatically regenerate stale metadata (locks and schemas) before pushing") + .option( + "--auto-metadata", + "Automatically regenerate stale metadata (locks and schemas) before pushing", + ) .option( "--accept-overriding-permissioned-as-with-self", "Accept that items with a different permissioned_as will be updated with your own user", diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 0129a4fd18..ebad7bc51c 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -78,6 +78,7 @@ export interface SyncOptions { skipResourceTypes?: boolean; skipSecrets?: boolean; skipWorkspaceDependencies?: boolean; + skipDatatableMigrations?: boolean; skipScripts?: boolean; skipFlows?: boolean; skipApps?: boolean; @@ -343,6 +344,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< | "includeSchedules" | "includeTriggers" | "skipWorkspaceDependencies" + | "skipDatatableMigrations" | "skipScripts" | "skipFlows" | "skipApps" @@ -375,6 +377,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< includeSettings: false, includeKey: false, skipWorkspaceDependencies: false, + skipDatatableMigrations: false, nonDottedPaths: false, syncBehavior: "v1", } as const; diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 59de21e2bf..a87206caac 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.775.2"; +export const VERSION = "1.777.1"; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 50ac0abc9f..5a2b599ea8 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -547,6 +547,11 @@ const result: wmill.S3Object = await wmill.writeS3File( Import: import * as wmill from 'windmill-client' +To know who is running the script, read the contextual variables rather than calling the API: +\`process.env.WM_END_USER_EMAIL || process.env.WM_EMAIL\`. WM_END_USER_EMAIL is the app viewer when +the run was triggered from an app and empty otherwise (both variables are always defined), WM_EMAIL +is the user the job is permissioned as. WM_USERNAME is the matching username. + workerHasInternalServer(): boolean /** @@ -577,11 +582,6 @@ async getResource(path?: string, undefinedIfEmpty?: boolean): Promise */ async getRootJobId(jobId?: string): Promise -/** - * @deprecated Use runScriptByPath or runScriptByHash instead - */ -async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise - /** * Run a script synchronously by its path and wait for the result * @param path - Script path in Windmill @@ -646,11 +646,6 @@ async getResult(jobId: string): Promise */ async getResultMaybe(jobId: string): Promise -/** - * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead - */ -async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise - /** * Run a script asynchronously by its path * @param path - Script path in Windmill @@ -703,13 +698,6 @@ getStatePath(): string */ async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise -/** - * Set the state - * @param state state to set - * @deprecated use setState instead - */ -async setInternalState(state: any): Promise - /** * Set the state * @param state state to set @@ -745,12 +733,6 @@ async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): P */ async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise -/** - * Get the internal state - * @deprecated use getState instead - */ -async getInternalState(): Promise - /** * Get the state shared across executions * @param path Optional state resource path override. Defaults to \`getStatePath()\`. @@ -886,15 +868,6 @@ async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ cancel: string; }> -/** - * @deprecated use getResumeUrls instead - */ -getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - /** * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) * @param audience audience of the token @@ -917,15 +890,6 @@ base64ToUint8Array(data: string): Uint8Array */ uint8ArrayToBase64(arrayBuffer: Uint8Array): string -/** - * Get email from workspace username - * This method is particularly useful for apps that require the email address of the viewer. - * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. - * @param username - * @returns email address - */ -async usernameToEmail(username: string): Promise - /** * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. * @@ -1347,6 +1311,11 @@ const result: wmill.S3Object = await wmill.writeS3File( Import: import * as wmill from 'windmill-client' +To know who is running the script, read the contextual variables rather than calling the API: +\`process.env.WM_END_USER_EMAIL || process.env.WM_EMAIL\`. WM_END_USER_EMAIL is the app viewer when +the run was triggered from an app and empty otherwise (both variables are always defined), WM_EMAIL +is the user the job is permissioned as. WM_USERNAME is the matching username. + workerHasInternalServer(): boolean /** @@ -1377,11 +1346,6 @@ async getResource(path?: string, undefinedIfEmpty?: boolean): Promise */ async getRootJobId(jobId?: string): Promise -/** - * @deprecated Use runScriptByPath or runScriptByHash instead - */ -async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise - /** * Run a script synchronously by its path and wait for the result * @param path - Script path in Windmill @@ -1446,11 +1410,6 @@ async getResult(jobId: string): Promise */ async getResultMaybe(jobId: string): Promise -/** - * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead - */ -async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise - /** * Run a script asynchronously by its path * @param path - Script path in Windmill @@ -1503,13 +1462,6 @@ getStatePath(): string */ async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise -/** - * Set the state - * @param state state to set - * @deprecated use setState instead - */ -async setInternalState(state: any): Promise - /** * Set the state * @param state state to set @@ -1545,12 +1497,6 @@ async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): P */ async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise -/** - * Get the internal state - * @deprecated use getState instead - */ -async getInternalState(): Promise - /** * Get the state shared across executions * @param path Optional state resource path override. Defaults to \`getStatePath()\`. @@ -1686,15 +1632,6 @@ async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ cancel: string; }> -/** - * @deprecated use getResumeUrls instead - */ -getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - /** * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) * @param audience audience of the token @@ -1717,15 +1654,6 @@ base64ToUint8Array(data: string): Uint8Array */ uint8ArrayToBase64(arrayBuffer: Uint8Array): string -/** - * Get email from workspace username - * This method is particularly useful for apps that require the email address of the viewer. - * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. - * @param username - * @returns email address - */ -async usernameToEmail(username: string): Promise - /** * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. * @@ -2241,6 +2169,11 @@ const result: wmill.S3Object = await wmill.writeS3File( Import: import * as wmill from 'windmill-client' +To know who is running the script, read the contextual variables rather than calling the API: +\`process.env.WM_END_USER_EMAIL || process.env.WM_EMAIL\`. WM_END_USER_EMAIL is the app viewer when +the run was triggered from an app and empty otherwise (both variables are always defined), WM_EMAIL +is the user the job is permissioned as. WM_USERNAME is the matching username. + workerHasInternalServer(): boolean /** @@ -2271,11 +2204,6 @@ async getResource(path?: string, undefinedIfEmpty?: boolean): Promise */ async getRootJobId(jobId?: string): Promise -/** - * @deprecated Use runScriptByPath or runScriptByHash instead - */ -async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false, tag: string | null = null): Promise - /** * Run a script synchronously by its path and wait for the result * @param path - Script path in Windmill @@ -2340,11 +2268,6 @@ async getResult(jobId: string): Promise */ async getResultMaybe(jobId: string): Promise -/** - * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead - */ -async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise - /** * Run a script asynchronously by its path * @param path - Script path in Windmill @@ -2397,13 +2320,6 @@ getStatePath(): string */ async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise -/** - * Set the state - * @param state state to set - * @deprecated use setState instead - */ -async setInternalState(state: any): Promise - /** * Set the state * @param state state to set @@ -2439,12 +2355,6 @@ async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): P */ async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise -/** - * Get the internal state - * @deprecated use getState instead - */ -async getInternalState(): Promise - /** * Get the state shared across executions * @param path Optional state resource path override. Defaults to \`getStatePath()\`. @@ -2580,15 +2490,6 @@ async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ cancel: string; }> -/** - * @deprecated use getResumeUrls instead - */ -getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - /** * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) * @param audience audience of the token @@ -2611,15 +2512,6 @@ base64ToUint8Array(data: string): Uint8Array */ uint8ArrayToBase64(arrayBuffer: Uint8Array): string -/** - * Get email from workspace username - * This method is particularly useful for apps that require the email address of the viewer. - * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. - * @param username - * @returns email address - */ -async usernameToEmail(username: string): Promise - /** * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. * @@ -3965,6 +3857,11 @@ result: S3Object = wmill.write_s3_file( Import: import wmill +To know who is running the script, read the contextual variables rather than calling the API: +\`os.environ.get("WM_END_USER_EMAIL") or os.environ.get("WM_EMAIL")\`. WM_END_USER_EMAIL is the app +viewer when the run was triggered from an app and empty otherwise (both variables are always +defined), WM_EMAIL is the user the job is permissioned as. WM_USERNAME is the matching username. + def worker_has_internal_server() -> bool def get_mocked_api() -> Optional[dict] @@ -4006,11 +3903,6 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response # New authentication token string def create_token(duration = dt.timedelta(days=1)) -> str -# Create a script job and return its job id. -# -# .. deprecated:: Use run_script_by_path_async or run_script_by_hash_async instead. -def run_script_async(path: str = None, hash_: str = None, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str - # Create a script job by path and return its job id. def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str @@ -4020,11 +3912,6 @@ def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: i # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str -# Run script synchronously and return its result. -# -# .. deprecated:: Use run_script_by_path or run_script_by_hash instead. -def run_script(path: str = None, hash_: str = None, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any - # Run script by path synchronously and return its result. def run_script_by_path(path: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False, tag: str = None) -> Any @@ -4411,11 +4298,6 @@ def get_approval_urls(step_key: str = 'approval', approver: str = None) -> dict # - The function checks for required environment variables (\`WM_FLOW_JOB_ID\`, \`WM_FLOW_STEP_ID\`) to ensure it is run in the appropriate context. def request_interactive_slack_approval(slack_resource_path: str, channel_id: str, message: str = None, approver: str = None, default_args_json: dict = None, dynamic_enums_json: dict = None) -> None -# Get email from workspace username -# This method is particularly useful for apps that require the email address of the viewer. -# Indeed, in the viewer context WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. -def username_to_email(username: str) -> str - # Send a message to a Microsoft Teams conversation with conversation_id, where success is used to style the message def send_teams_message(conversation_id: str, text: str, success: bool = True, card_block: dict = None) @@ -4449,6 +4331,18 @@ def get_workspace() -> str def get_version() -> str +# Create a script job and return its job ID. +# +# Args: +# hash_or_path: Script hash or path (determined by presence of '/') +# args: Script arguments +# scheduled_in_secs: Delay before execution in seconds +# tag: Override the worker tag the job runs on +# +# Returns: +# Job ID string +def run_script_async(hash_or_path: str, args: Dict[str, Any] = None, scheduled_in_secs: int = None, tag: str = None) -> str + # Run a script synchronously by hash and return its result. # # Args: @@ -5486,7 +5380,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -6840,6 +6734,7 @@ app related commands - \`--host \` - Host to bind the dev server to - \`--entry \` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) - \`--no-open\` - Don't automatically open the browser + - \`--recording\` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development - \`app lint [app_folder:string]\` - Lint a raw app folder to validate structure and buildability - \`--fix\` - Attempt to fix common issues (not implemented yet) - \`app new\` - create a new raw app from a template diff --git a/cli/src/types.ts b/cli/src/types.ts index f0cd58abbb..6dd2e6ff77 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -5,6 +5,7 @@ import * as path from "node:path"; import { sep as SEP } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseContent } from "./utils/yaml.ts"; +import { isDbtDescriptorPath } from "./utils/resource_folders.ts"; import { pushApp } from "./commands/app/app.ts"; import { pushFolder } from "./commands/folder/folder.ts"; import { pushFlow } from "./commands/flow/flow.ts"; @@ -174,6 +175,21 @@ function redactString(s: string): string { return s.slice(0, 5) + "*".repeat(s.length - 5); } +export interface PushObjOptions { + /** Optional commit/update message */ + message?: string; + /** The original local file path (used for branch-specific resource file resolution) */ + originalLocalPath?: string; + /** Identity to attribute the push to, for the types that carry one */ + permissionedAsContext?: PermissionedAsContext; + /** Whether the item is workspace-specific */ + wsSpecific?: boolean; + /** encryption_key push: non-interactive flag and explicit re-encryption choice */ + keyPushOpts?: PushWorkspaceKeyOptions; + /** TypeScript runtime a bare `.ts` denotes, for raw-app runnables */ + defaultTs?: "bun" | "deno"; +} + /** * Pushes an object to the workspace server based on its type * @param workspace - The workspace ID to push to @@ -182,9 +198,7 @@ function redactString(s: string): string { * @param newObj - The new object state to push * @param plainSecrets - Whether to store secrets in plain text * @param alreadySynced - Array to track already synced items - * @param message - Optional commit/update message - * @param originalLocalPath - The original local file path (used for branch-specific resource file resolution) - * @param keyPushOpts - Options for the encryption_key push: non-interactive flag and explicit re-encryption choice + * @param opts - Per-type extras; see PushObjOptions */ export async function pushObj( workspace: string, @@ -193,12 +207,16 @@ export async function pushObj( newObj: any, plainSecrets: boolean, alreadySynced: string[], - message?: string, - originalLocalPath?: string, - permissionedAsContext?: PermissionedAsContext, - wsSpecific?: boolean, - keyPushOpts?: PushWorkspaceKeyOptions, + opts: PushObjOptions = {}, ) { + const { + message, + originalLocalPath, + permissionedAsContext, + wsSpecific, + keyPushOpts, + defaultTs, + } = opts; const typeEnding = getTypeStrFromPath(p); if (typeEnding === "app") { @@ -212,7 +230,7 @@ export async function pushObj( if (!rawAppName) { throw new Error(`Could not extract raw app name from path: ${p}`); } - await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message); + await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs); } else if (typeEnding === "folder") { await pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { @@ -388,7 +406,11 @@ export function getTypeStrFromPath( parsed.ext == ".rb" || parsed.ext == ".r" || // for related places search: ADD_NEW_LANG - (parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook") + (parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook") || + // A dbt descriptor is `__dbt/wm_dbt.yaml`. Without this it reads + // as one of the CLI's own `.yaml` metadata files and a pull writes the + // script's metadata and lock but never its content. + isDbtDescriptorPath(p) ) { return "script"; } diff --git a/cli/src/utils/git.ts b/cli/src/utils/git.ts index da1ac14414..5ca06f427a 100644 --- a/cli/src/utils/git.ts +++ b/cli/src/utils/git.ts @@ -75,6 +75,114 @@ export function getWorkspaceIdForWorkspaceForkFromBranchName(branchName: string) return `${WM_FORK_PREFIX}-${branchName.slice(start)}` } +/** + * Whether this checkout can vouch for what it once held under `migrations/datatable/**`. + * + * `known` lists every such path recorded on the current branch (paths stay listed after + * the commit that removed them, so a real deletion is still recognisable). `unknown` + * means a file's absence from the working tree proves nothing, either because the + * history can't be read — no repository, a shallow clone's truncated history, an + * unresolvable root, a failing git — or because the working tree deliberately doesn't + * mirror it, as in a sparse checkout. Both shapes carry the same obligation on the + * caller: trust nothing, rather than read absence as evidence. + */ +export type RecordedMigrationPaths = + | { kind: "known"; paths: Set } + | { kind: "unknown"; reason: string; remedy: string }; + +/** + * Resolve [`RecordedMigrationPaths`] for the checkout at the current directory. + * + * This is the durable answer to "did this checkout ever track that migration?", which + * the working tree cannot give: an absent `migrations/datatable/` is equally a clone + * that never pulled migrations and one where the last was deleted, and creating a + * migration locally (`wmill datatable migrate new`) makes the directory appear without + * anything having been tracked. + * + * Scoped to `HEAD`, not `--all`: a migration that only ever existed on some other + * branch is not evidence that *this* branch ever tracked it, and using it as such would + * authorize deleting it. Output paths are repo-root-relative, so the `--show-prefix` of + * the working directory is stripped to match the cwd-relative paths a sync diff uses. + */ +export function gitRecordedDatatableMigrationPaths(): RecordedMigrationPaths { + if (!isGitRepository()) { + return { + kind: "unknown", + reason: "this directory is not a git repository", + remedy: "Run the push from a git checkout of the synced repository", + }; + } + const shallow = spawnSync("git", ["rev-parse", "--is-shallow-repository"], { + encoding: "utf8", + stdio: "pipe", + }); + if ((shallow.stdout ?? "").trim() === "true") { + return { + kind: "unknown", + reason: "this is a shallow clone, so its history is truncated", + remedy: + "Fetch the full history (for actions/checkout, fetch-depth: 0)", + }; + } + // A sparse checkout can record migrations in history while never materialising + // them in the working tree, so their absence there says nothing about whether the + // user deleted them. Over-protective for a sparse cone that does include + // migrations/, which the interactive prompt can still override. + // `--type=bool` normalises git's booleans (1, yes, on, …) to true/false; a raw + // `--get` would let `core.sparseCheckout = 1` walk straight past this. + const sparse = spawnSync( + "git", + ["config", "--type=bool", "--get", "core.sparseCheckout"], + { encoding: "utf8", stdio: "pipe" }, + ); + if ((sparse.stdout ?? "").trim() === "true") { + return { + kind: "unknown", + reason: + "this is a sparse checkout, so its working tree may not hold every tracked file", + remedy: "Run the push from a full (non-sparse) checkout", + }; + } + const prefixOut = spawnSync("git", ["rev-parse", "--show-prefix"], { + encoding: "utf8", + stdio: "pipe", + }); + if ((prefixOut.status ?? 1) !== 0) { + return { + kind: "unknown", + reason: "the repository root could not be resolved", + remedy: "Check that git runs correctly in this directory", + }; + } + const prefix = (prefixOut.stdout ?? "").trim(); + + const r = spawnSync( + "git", + ["log", "HEAD", "--format=", "--name-only", "--", "migrations/datatable"], + { encoding: "utf8", stdio: "pipe", maxBuffer: 64 * 1024 * 1024 }, + ); + if ((r.status ?? 1) !== 0) { + log.debug(`Could not read git history for migrations: ${r.stderr ?? ""}`); + return { + kind: "unknown", + reason: "its history could not be read", + remedy: "Check that git runs correctly in this directory", + }; + } + const paths = new Set(); + for (const line of (r.stdout ?? "").split("\n")) { + const p = line.trim(); + if (p.length === 0) continue; + if (prefix.length > 0) { + if (!p.startsWith(prefix)) continue; + paths.add(p.slice(prefix.length)); + } else { + paths.add(p); + } + } + return { kind: "known", paths }; +} + export function isGitRepository(): boolean { try { execSync("git rev-parse --git-dir", { @@ -319,6 +427,10 @@ export function gitSyncIncludePattern( return `${path}.azure_trigger.*`; case "emailtrigger": return `${path}.email_trigger.*`; + case "datatable_migration": + // One migration is two files under `migrations/datatable/
/`; the + // backend already sends the repo-relative base path. + return `${path}.up.sql,${path}.down.sql`; default: // Scripts: `${path}.*` matches the dotted layout // (`${path}.script.yaml` etc.), `${path}__mod/**` matches the folder diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index ebb35e9d33..1905121c3a 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -19,9 +19,10 @@ import { languageNeedsLock, } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; -import { getModuleFolderSuffix, isModuleEntryPoint, scriptPathToRemotePath } from "./resource_folders.ts"; +import { dbtGeneratedDirs, isUnderGeneratedDir, isBundledModuleFile, getModuleFolderSuffix, isModuleEntryPoint, scriptPathToRemotePath } from "./resource_folders.ts"; import { findCodebase, yamlOptions } from "../commands/sync/sync.ts"; import { generateHash, readInlinePathSync, getHeaders, readTextFile, readTextFileSync } from "./utils.ts"; +import { DBT_DESCRIPTOR_NAME, isMissingDbtDescriptor } from "./resource_folders.ts"; import { detectAuthGatewayChallenge } from "./http_guards.ts"; import { SyncCodebase } from "./codebase.ts"; @@ -218,6 +219,13 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); + // Whether the metadata and lock live INSIDE that folder. They do for a `__mod` + // bundle, whose folder is Windmill's. A dbt project's folder is dbt's, taken + // verbatim, so its companions stay beside it — writing them in would leave + // stray `script.yaml`/`script.lock` files in the deployed project and leave + // the metadata sync actually reads untouched. + const metadataInFolder = isFolderLayout && language !== "dbt"; + // For folder layout, parseMetadataFile is called with remotePath which // will find __mod/script.yaml via the folder layout fallback const metadataWithType = await parseMetadataFile( @@ -225,8 +233,12 @@ export async function generateScriptMetadataInternal( undefined, ); - // read script content - const scriptContent = await readTextFile(scriptPath); + // read script content — a dbt project's descriptor is optional, and absent + // means an empty descriptor rather than a script that cannot be pushed. + const scriptContent = await readTextFile(scriptPath).catch((e) => { + if (isMissingDbtDescriptor(scriptPath, e)) return ""; + throw e; + }); const metadataContent = await readTextFile(metadataWithType.path); const filteredRawWorkspaceDependencies = filterWorkspaceDependencies( @@ -238,7 +250,8 @@ export async function generateScriptMetadataInternal( // Compute the module folder path early so we can include module hashes in stale check const moduleFolderPath = isFolderLayout ? path.dirname(scriptPath) - : scriptPath.substring(0, scriptPath.indexOf(".")) + getModuleFolderSuffix(); + : scriptPath.substring(0, scriptPath.indexOf(".")) + + getModuleFolderSuffix(language); const hasModules = existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory(); @@ -250,7 +263,8 @@ export async function generateScriptMetadataInternal( let moduleHashes: Record = {}; if (hasModules) { moduleHashes = await computeModuleHashes( - moduleFolderPath, opts.defaultTs, tree ? {} : rawWorkspaceDependencies, isFolderLayout + moduleFolderPath, opts.defaultTs, tree ? {} : rawWorkspaceDependencies, isFolderLayout, + language === "dbt", ); } const hasModuleHashes = Object.keys(moduleHashes).length > 0; @@ -349,7 +363,7 @@ export async function generateScriptMetadataInternal( if (!hasCodebase) { const tempScriptRefs = tree?.getTempScriptRefs(remotePath); - const lockPathOverride = isFolderLayout + const lockPathOverride = metadataInFolder ? path.dirname(scriptPath) + "/script.lock" : undefined; await updateScriptLock( @@ -366,8 +380,13 @@ export async function generateScriptMetadataInternal( metadataParsedContent.lock = ""; } - // Generate locks for modules in __mod/ folder - if (hasModules) { + // Generate locks for modules in __mod/ folder. + // + // Never for a dbt bundle: its files are the project's own SQL and YAML, none + // of which is a Windmill script needing a lockfile, and writing `foo.lock` + // beside `foo.sql` puts our artifacts inside a tree we promise to round-trip + // byte-for-byte. + if (hasModules && language !== "dbt") { // Identify which modules changed by comparing per-module hashes let changedModules: string[] | undefined; if (hasModuleHashes) { @@ -387,7 +406,7 @@ export async function generateScriptMetadataInternal( ); } } else { - if (isFolderLayout) { + if (metadataInFolder) { metadataParsedContent.lock = "!inline " + remotePath.replaceAll(SEP, "/") + getModuleFolderSuffix() + "/script.lock"; } else { @@ -399,7 +418,7 @@ export async function generateScriptMetadataInternal( // Write metadata back to the correct path let metaPath: string; let newMetadataContent: string; - if (isFolderLayout) { + if (metadataInFolder) { if (metadataWithType.isJson) { metaPath = path.dirname(scriptPath) + "/script.json"; newMetadataContent = JSON.stringify(metadataParsedContent); @@ -765,15 +784,18 @@ async function updateScriptLock( tempScriptRefs?: Record, lockPathOverride?: string, ): Promise { - if ( - !( - (workspaceDependenciesLanguages.some((l) => l.language == language) && - language !== "powershell") || - language == "deno" || - language == "rust" || - language == "ansible" - ) - ) { + if (!languageNeedsLock(language)) { + // A dbt lock is written by the dependency job on a worker, from a real + // `dbt deps`/`dbt parse`, so there is nothing to generate here. Restore the + // reference to the file `wmill sync pull` wrote: the caller has already + // resolved it into the metadata, and leaving it resolved inlines the lock + // into the yaml on every run. + if (language === "dbt") { + const lockPath = lockPathOverride ?? remotePath + ".script.lock"; + if (existsSync(lockPath)) { + metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/"); + } + } return; } @@ -995,6 +1017,9 @@ export async function inferSchema( } else if (language === "rlang") { const { parse_r } = await loadParser("windmill-parser-wasm-r"); inferedSchema = JSON.parse(parse_r(content)); + } else if (language === "dbt") { + const { parse_dbt } = await loadParser("windmill-parser-wasm-yaml"); + inferedSchema = JSON.parse(parse_dbt(content)); // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); @@ -1368,8 +1393,15 @@ async function computeModuleHashes( defaultTs: "bun" | "deno" | undefined, rawWorkspaceDependencies: Record, isFolderLayout: boolean, + // A dbt project's files are taken verbatim, so hash them the same way the + // push reads them: a `.sql` model has no inferable language, and dropping it + // here would leave an edited model looking up to date. + verbatim: boolean = false, ): Promise> { const hashes: Record = {}; + const skipDirs = verbatim + ? dbtGeneratedDirs(moduleFolderPath) + : new Set(); async function readDir(dirPath: string, relPrefix: string) { const entries = readdirSync(dirPath, { withFileTypes: true }); @@ -1379,15 +1411,31 @@ async function computeModuleHashes( const isTopLevel = relPrefix === ""; if (entry.isDirectory()) { + // A configured `target-path` may be nested (`build/target`), so the + // comparison is on the project-relative path, not the entry name. + if (skipDirs.size > 0 && isUnderGeneratedDir(relPath, skipDirs)) continue; await readDir(fullPath, relPath); + // See the bundle builder: a verbatim (dbt) bundle carries a `.lock` the + // project authored, so the hash has to see it or a change to it would + // never be detected as a change. } else if ( entry.isFile() && - !entry.name.endsWith(".lock") && - !(isFolderLayout && isTopLevel && entry.name.startsWith("script.")) + (verbatim || !entry.name.endsWith(".lock")) && + !(isFolderLayout && isTopLevel && entry.name.startsWith("script.")) && + // The descriptor is the script's CONTENT, hashed as such: counting it + // here too would make one edit look like two changes. + !(verbatim && isTopLevel && entry.name === DBT_DESCRIPTOR_NAME) ) { - try { - inferContentTypeFromFilePath(entry.name, defaultTs); - } catch { + if (!verbatim) { + try { + inferContentTypeFromFilePath(entry.name, defaultTs); + } catch { + continue; + } + } else if (!isBundledModuleFile(fullPath)) { + // Hash only what the push actually sends. Hashing a file the bundle + // drops would make the script permanently stale: every check would + // see a change no push can ever resolve. continue; } const content = readTextFileSync(fullPath); diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index 653c443935..04ed260f07 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -8,6 +8,7 @@ * (.flow, .app, .raw_app) or dunder-prefixed names (__flow, __app, __raw_app). */ +import { existsSync } from "node:fs"; import * as log from "../core/log.ts"; import { sep as SEP } from "node:path"; import { yamlParseFile } from "./yaml.ts"; @@ -503,28 +504,323 @@ export function isFlowFolderMetadataFile(p: string): boolean { * to avoid confusion with file extensions. */ const MODULE_SUFFIX = "__mod"; +/** dbt scripts carry a whole dbt project, not helper code. The folder says so, + * and it is what a dbt developer points `--project-dir` at. */ +export const DBT_MODULE_SUFFIX = "__dbt"; +const MODULE_SUFFIXES = [MODULE_SUFFIX, DBT_MODULE_SUFFIX]; + +/** A dbt project's descriptor, inside the project it configures and OPTIONAL: + * an unmodified dbt project is already a complete Windmill script, and this + * file only appears when one needs something Windmill-specific (run arguments, + * a named warehouse, an engine pin). Its absence is an empty descriptor, never + * a missing script. */ +export const DBT_DESCRIPTOR_NAME = "wm_dbt.yaml"; + +/** Where a dbt script's descriptor lives, given its base path. */ +export function dbtDescriptorPath(scriptBasePath: string): string { + return scriptBasePath + DBT_MODULE_SUFFIX + "/" + DBT_DESCRIPTOR_NAME; +} + +/** Whether an error is a dbt descriptor that simply is not there. */ +export function isMissingDbtDescriptor(filePath: string, e: unknown): boolean { + if ((e as { code?: string })?.code !== "ENOENT") return false; + const norm = filePath.replaceAll("\\", "/"); + if (!isDbtDescriptorPath(norm)) return false; + // And the PROJECT is there. Absent both, this is not a descriptor-less + // project but a path that does not exist — a typo, or a project someone + // deleted — and treating it as an empty descriptor pushes `modules: + // undefined` over a deployed bundle, dropping it while reporting success. + return existsSync( + norm.slice(0, -DBT_DESCRIPTOR_NAME.length) + "dbt_project.yml" + ); +} + +/** Whether a path is a dbt descriptor, i.e. a dbt script's content file. */ +export function isDbtDescriptorPath(p: string): boolean { + const norm = normalizeSep(p); + const base = getScriptBasePathFromModulePath(norm); + return base !== undefined && norm === dbtDescriptorPath(base); +} /** - * Get the module folder suffix (always "__mod") + * Module folder suffix for a script: `__dbt` for a dbt project, `__mod` + * otherwise. */ -export function getModuleFolderSuffix(): string { - return MODULE_SUFFIX; +export function getModuleFolderSuffix(language?: string): string { + return language === "dbt" ? DBT_MODULE_SUFFIX : MODULE_SUFFIX; } /** * Check if a path is inside a script module folder. - * Matches patterns like: .../my_script__mod/... + * Matches patterns like: .../my_script__mod/... or .../my_project__dbt/... */ export function isScriptModulePath(p: string): boolean { - return normalizeSep(p).includes(MODULE_SUFFIX + "/"); + const n = normalizeSep(p); + return MODULE_SUFFIXES.some((suffix) => n.includes(suffix + "/")); +} + +/** Per-file ceiling for a dbt project's bundle. Real dbt code is small (about + * 500 bytes median, 1.9 KB at p90 measured across dbt_utils), so this only + * ever catches a committed dataset, which belongs in the warehouse rather than + * in every version of the script. */ +export const MAX_MODULE_BYTES = 5 * 1024 * 1024; + +/** + * Whether a dbt project file is one the bundle carries. + * + * A dbt project's authored files are text. A binary one -- an image under + * `docs/`, a `.DS_Store`, a parquet seed -- would be read as mojibake and, if + * it carries a NUL, rejected by Postgres with an opaque `unsupported Unicode + * escape sequence`. Binary is detected the way `git` does it, by a NUL in the + * first 8000 bytes, rather than by extension, which `docs/` and stray dotfiles + * do not follow. + * + * The push, the staleness hash and the sync diff all ask this same question: a + * file one drops and another keeps is a change no push can ever resolve. + */ +export function isBundledModuleFile(fullPath: string): boolean { + return moduleFileExclusion(fullPath) === undefined; +} + +/** + * WHY the bundle does not carry a file, when it does not. + * + * The two reasons are not interchangeable. `binary` is dbt's own leftovers and + * stray archives: nothing to say, so sync hides them. `oversized` is a file the + * project authored and dbt WOULD read — a large seed CSV — so it has to stay + * visible in the diff, or the push that reports the actionable size error never + * runs and the remote project is silently left incomplete. + */ +export function moduleFileExclusion( + fullPath: string, +): "binary" | "oversized" | undefined { + // Size from `stat` and only the first 8 KB read: a project may sit next to a + // multi-gigabyte parquet seed or a stray archive, and reading one whole just + // to classify it would stall the sync or exhaust the CLI. + let size: number; + let fd: number; + try { + size = fs.statSync(fullPath).size; + fd = fs.openSync(fullPath, "r"); + } catch { + // Unreadable is not the same as excluded. A pull asks this about files that + // do not exist locally yet, and answering "not carried" there would make + // sync ignore the whole incoming project and write nothing. + return undefined; + } + let binary: boolean; + try { + const head = Buffer.alloc(8000); + const read = fs.readSync(fd, head, 0, 8000, 0); + binary = head.subarray(0, read).includes(0); + } catch { + return undefined; + } finally { + fs.closeSync(fd); + } + if (binary) return "binary"; + return size > MAX_MODULE_BYTES ? "oversized" : undefined; +} + +/** + * The refusal an oversized dbt project file earns, raised WITHOUT reading it. + * + * dbt would have read the file, so deploying the project without it ships + * something that compiles here and fails at run time with a missing relation — + * hence an error rather than a skip. Every path that would otherwise load the + * body (the sync map, the push) asks first: a multi-gigabyte seed must not be + * buffered just to be refused. + */ +export function oversizedModuleFileError(relPath: string, size: number): Error { + return new Error( + `${relPath} is ${Math.ceil(size / 1024 / 1024)} MB, over the ` + + `${MAX_MODULE_BYTES / 1024 / 1024} MB per-file limit for a dbt project file. ` + + `Deploying without it would leave the project incomplete — shrink the file, or ` + + `keep it out of the project folder.`, + ); +} + +/** + * Refuse an oversized dbt project file before its content is read. `undefined` + * for everything else, including binary files the bundle merely drops. + */ +export function oversizedDbtFileError( + fullPath: string, + relPath: string, +): Error | undefined { + if (!isDbtModulePath(relPath)) return undefined; + if (moduleFileExclusion(fullPath) !== "oversized") return undefined; + let size = 0; + try { + size = fs.statSync(fullPath).size; + } catch { + return undefined; + } + return oversizedModuleFileError(relPath, size); +} + +/** Whether a path is inside a dbt project's module folder specifically: those + * files are taken verbatim, with no language inference. */ +export function isDbtModulePath(p: string): boolean { + // The OUTERMOST boundary decides, like `getScriptBasePathFromModulePath`. + // Scanning anywhere in the path would call `foo__mod/vendor/x__dbt/a.ts` a dbt + // project file, and the push would then look for `foo.script.yaml` instead of + // the ordinary module entry point and skip the edit. + const norm = normalizeSep(p); + const base = getScriptBasePathFromModulePath(norm); + return base !== undefined && norm.startsWith(base + DBT_MODULE_SUFFIX + "/"); +} + +/** dbt writes these; a project authors them nowhere. Importing a stale + * `target/` would ship a manifest this runtime then reads as the graph, and + * `dbt_packages/` is a vendored copy the worker restores from its own cache. */ +const DBT_GENERATED_DIRS = ["target", "dbt_packages", "logs", ".git", ".venv", "__pycache__"]; + +const dbtGeneratedDirsCache = new Map< + string, + { stamp: string; dirs: Set } +>(); + +/** `{{ env_var('NAME') }}` / `{{ env_var("NAME", "default") }}`. */ +const DBT_ENV_VAR_CALL = + /\{\{\s*env_var\(\s*['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]*)['"]\s*)?\)\s*\}\}/g; + +/** + * Render `dbt_project.yml`'s own `env_var()` calls, which dbt allows there too. + * A directory setting left as its template names no directory on disk, so the + * generated tree it points at would be bundled as project source. + * + * Against `process.env`, because the CLI runs where the project was built: that + * is the environment dbt used to produce the tree being read. + */ +export function renderDbtEnvVars(value: string): string { + return value.replace( + DBT_ENV_VAR_CALL, + (whole, name: string, fallback: string | undefined) => + process.env[name] ?? fallback ?? whole, + ); +} + +/** + * Files that keep a project's secrets next to it rather than in it: dbt reads + * none of them (`env_var()` takes the process environment), and the documented + * way into a bundle is `cp -r my-project/.`, which copies whatever the checkout + * holds — including the `.env` a `.gitignore` was keeping out of the repo. + */ +export function isLocalSecretFile(name: string): boolean { + return name === ".env" || name.startsWith(".env.") || name === ".envrc"; +} + +/** + * Directories to leave out of a dbt project's module bundle, as project-relative + * paths — `target-path` and friends may be nested (`build/target`). + * + * `target-path`, `packages-install-path` and `clean-targets` are configurable, + * so they are read from the project rather than assumed. Cached per project + * folder: this is called once per file of a sync. + */ +export function dbtGeneratedDirs(moduleFolderPath: string): Set { + const projectFile = path.join(moduleFolderPath, "dbt_project.yml"); + // Cached against the project file's identity, not merely its folder: `wmill + // dev` is a long-running process, so a `target-path` edited mid-session would + // otherwise keep excluding the old directory and start bundling the new one + // as project source. One entry per folder, replaced when the file changes. + let stamp = ""; + try { + const st = fs.statSync(projectFile); + stamp = `${st.mtimeMs}:${st.size}`; + } catch { + // No project file yet: the defaults apply, and "absent" is its own stamp. + } + const cached = dbtGeneratedDirsCache.get(moduleFolderPath); + if (cached && cached.stamp === stamp) return cached.dirs; + const dirs = new Set(DBT_GENERATED_DIRS); + const add = (raw: string) => { + const v = normalizeSep(renderDbtEnvVars(raw).trim().replace(/^["']|["']$/g, "")) + .replace(/^\.\//, "") + .replace(/\/+$/, ""); + // A configured path that escapes the project is dbt's problem, not ours; + // ignoring it here just means those files stay in the bundle. + if (v && !v.startsWith("/") && !v.split("/").includes("..")) dirs.add(v); + }; + try { + const projectYml = fs.readFileSync(projectFile, "utf-8"); + for (const m of projectYml.matchAll( + /^\s*(?:target-path|packages-install-path)\s*:\s*([^\n#]+)/gm, + )) { + add(m[1]); + } + // `clean-targets` in either of dbt's two spellings: inline `[a, b]`, and the + // block form, whose entries are on the lines that follow. + const lines = projectYml.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const head = lines[i].match(/^\s*clean-targets\s*:\s*(.*)$/); + if (!head) continue; + const inline = head[1].match(/^\[([^\]]*)\]/); + if (inline) { + inline[1].split(",").forEach(add); + continue; + } + for (let j = i + 1; j < lines.length; j++) { + const item = lines[j].match(/^\s+-\s*([^\n#]+)$/); + if (!item) break; + add(item[1]); + } + } + } catch { + // No dbt_project.yml yet (a descriptor pushed before its project): the + // defaults still apply. + } + dbtGeneratedDirsCache.set(moduleFolderPath, { stamp, dirs }); + return dirs; +} + +/** + * Whether a project-relative path sits inside one of `dirs`. Compared segment + * by segment: `targetx/a` must not match a configured `target`. + */ +export function isUnderGeneratedDir(rel: string, dirs: Set): boolean { + const n = normalizeSep(rel); + for (const d of dirs) { + if (n === d || n.startsWith(d + "/")) return true; + } + return false; +} + +/** + * Whether a path under a `__dbt/` folder is one dbt generated rather than one + * the project authors. Those never belong to the bundle, so sync must not offer + * them as items of their own either. + */ +export function isDbtGeneratedPath(p: string): boolean { + const n = normalizeSep(p); + // Anchored on the outermost boundary, like every other helper here. Matching + // `__dbt/` anywhere would call `foo__mod/vendor/x__dbt/target/a.ts` generated + // dbt output, and `ignoreF` would then exclude an ordinary module file so a + // module-only edit never deploys its parent script. + if (!isDbtModulePath(n)) return false; + const base = getScriptBasePathFromModulePath(n)!; + const projectRoot = base + DBT_MODULE_SUFFIX; + const rel = n.slice(projectRoot.length + 1); + if (isUnderGeneratedDir(rel, dbtGeneratedDirs(projectRoot))) { + return true; + } + // Not generated, but not carried either: the bundle drops it, so the diff + // must not keep offering it as a pending change. An OVERSIZED one is the + // exception — see `moduleFileExclusion`: hiding it here is what would make an + // edit to a large seed report no change at all. + return ( + isLocalSecretFile(n.slice(n.lastIndexOf("/") + 1)) || + moduleFileExclusion(p) === "binary" + ); } /** * Build the module folder path from a script's base path (without extension). - * e.g., "f/my_script" -> "f/my_script__mod" + * e.g., "f/my_script" -> "f/my_script__mod", or "__dbt" for a dbt project. */ -export function buildModuleFolderPath(scriptBasePath: string): string { - return scriptBasePath + MODULE_SUFFIX; +export function buildModuleFolderPath(scriptBasePath: string, language?: string): string { + return scriptBasePath + getModuleFolderSuffix(language); } /** @@ -533,10 +829,21 @@ export function buildModuleFolderPath(scriptBasePath: string): string { */ export function isModuleEntryPoint(p: string): boolean { const norm = normalizeSep(p); + // Anchored on the OUTERMOST module boundary, like + // `getScriptBasePathFromModulePath`. Scanning for `__mod/` alone would match a + // `legacy__mod/` directory nested inside a dbt project — dbt owns those names + // verbatim — and call its `script.ts` this script's entry point. + const base = getScriptBasePathFromModulePath(norm); + if (base === undefined) return false; + // A dbt project's entry point is its descriptor, which sits INSIDE the + // project so that an author writes nothing outside the directory dbt itself + // reads. + if (norm.startsWith(base + DBT_MODULE_SUFFIX + "/")) { + return norm === dbtDescriptorPath(base); + } const suffix = MODULE_SUFFIX + "/"; - const idx = norm.indexOf(suffix); - if (idx === -1) return false; - const rest = norm.slice(idx + suffix.length); + if (!norm.startsWith(base + suffix)) return false; + const rest = norm.slice(base.length + suffix.length); return rest.startsWith("script.") && !rest.includes("/"); } @@ -544,13 +851,21 @@ export function isModuleEntryPoint(p: string): boolean { * Extract the script base path from a module folder entry. * e.g., "u/admin/my_script__mod/script.ts" -> "u/admin/my_script" * e.g., "u/admin/my_script__mod/helper.ts" -> "u/admin/my_script" + * e.g., "f/x/proj__dbt/models/a.sql" -> "f/x/proj" */ export function getScriptBasePathFromModulePath(p: string): string | undefined { const norm = normalizeSep(p); - const suffix = MODULE_SUFFIX + "/"; - const idx = norm.indexOf(suffix); - if (idx === -1) return undefined; - return norm.slice(0, idx); + // The OUTERMOST boundary, not the first suffix that happens to match. A dbt + // project's directories are the author's verbatim, so `foo__dbt/models/ + // legacy__mod/a.sql` is legal — taking `__mod` first would call + // `foo__dbt/models/legacy` the script and look for a descriptor that is not + // there, silently skipping the deploy. + let best: number | undefined; + for (const suffix of MODULE_SUFFIXES) { + const idx = norm.indexOf(suffix + "/"); + if (idx !== -1 && (best === undefined || idx < best)) best = idx; + } + return best === undefined ? undefined : norm.slice(0, best); } /** diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index a713e41782..318f84b7c9 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -1,3 +1,4 @@ +import { isDbtDescriptorPath } from "./resource_folders.ts"; export type ScriptLanguage = | "python3" | "deno" @@ -21,6 +22,7 @@ export type ScriptLanguage = | "ansible" | "ruby" | "rlang" + | "dbt" | "java"; // for related places search: ADD_NEW_LANG @@ -106,6 +108,8 @@ export function inferContentTypeFromFilePath( return "java"; } else if (contentPath.endsWith(".rb")) { return "ruby"; + } else if (isDbtDescriptorPath(contentPath)) { + return "dbt"; } else if (contentPath.endsWith(".r")) { return "rlang"; // for related places search: ADD_NEW_LANG @@ -119,7 +123,7 @@ export function inferContentTypeFromFilePath( throw new Error( `Cannot infer script language from extension '${ext}' (file ${contentPath}).` + hint + - "\nSupported extensions: .ts (bun/deno), .py, .go, .sh, .ps1, .php, .rs, .cs, .nu, .java, .rb, .r, .gql, .playbook.yml, .pg.sql, .my.sql, .bq.sql, .sf.sql, .ms.sql, .odb.sql, .duckdb.sql" + "\nSupported extensions: .ts (bun/deno), .py, .go, .sh, .ps1, .php, .rs, .cs, .nu, .java, .rb, .r, .gql, .playbook.yml, .pg.sql, .my.sql, .bq.sql, .sf.sql, .ms.sql, .odb.sql, .duckdb.sql, and a dbt project folder `__dbt/`" ); } } diff --git a/cli/test/datatable_migrations_unit.test.ts b/cli/test/datatable_migrations_unit.test.ts index 09af49f890..2db9371629 100644 --- a/cli/test/datatable_migrations_unit.test.ts +++ b/cli/test/datatable_migrations_unit.test.ts @@ -15,6 +15,7 @@ import * as path from "node:path"; import * as os from "node:os"; import { parseDatatableMigrationPath } from "../src/types.ts"; import { validateLocalMigrations } from "../src/commands/datatable_migrations.ts"; +import { untrackedDatatableMigrationDeletions } from "../src/commands/sync/sync.ts"; describe("parseDatatableMigrationPath", () => { test("parses up and down files of the new layout", () => { @@ -120,3 +121,61 @@ describe("validateLocalMigrations", () => { expect(validateLocalMigrations()).toEqual([]); }); }); + +// ============================================================================= +// untrackedDatatableMigrationDeletions — the push-side safety net. +// +// Migrations bypass the repo's path filters, so a clone made before they were +// synced sees every server-side migration as remote-only, and +// `pushMigrationFromDisk` reads a missing `.up.sql` as "delete it". What the repo +// has ever committed under migrations/datatable/ is the durable answer to "did we +// track this?" — the working tree is not, because creating a migration locally +// makes the directory appear without anything having been tracked. +// ============================================================================= + +describe("untrackedDatatableMigrationDeletions", () => { + const A_UP = "migrations/datatable/mydb/20260101000000_a.up.sql"; + const A_DOWN = "migrations/datatable/mydb/20260101000000_a.down.sql"; + const changes = [ + { name: "deleted", path: A_UP }, + { name: "deleted", path: A_DOWN }, + { name: "deleted", path: "f/foo/bar.script.yaml" }, + { name: "added", path: "migrations/datatable/mydb/20260102000000_b.up.sql" }, + ]; + + test("trusts a deletion the repository has committed before", () => { + expect( + untrackedDatatableMigrationDeletions(changes, { kind: "known", paths: new Set([A_UP, A_DOWN]) }), + ).toEqual([]); + }); + + test("flags migrations this repository has never recorded", () => { + expect( + untrackedDatatableMigrationDeletions(changes, { kind: "known", paths: new Set() }).map((c) => c.path), + ).toEqual([A_UP, A_DOWN]); + }); + + test("a locally created migration does not vouch for unrelated ones", () => { + // `wmill datatable migrate new` makes migrations/datatable/ exist without the + // checkout having tracked anything, so only the recorded paths count. + const recorded = { + kind: "known" as const, + paths: new Set(["migrations/datatable/mydb/20260102000000_b.up.sql"]), + }; + expect( + untrackedDatatableMigrationDeletions(changes, recorded).map((c) => c.path), + ).toEqual([A_UP, A_DOWN]); + }); + + test("trusts nothing when the history cannot be consulted", () => { + // A shallow clone or a non-repository can't prove a path was never tracked, + // so absence is not read as permission to delete. + expect( + untrackedDatatableMigrationDeletions(changes, { + kind: "unknown", + reason: "this is a shallow clone, so its history is truncated", + remedy: "Fetch the full history (for actions/checkout, fetch-depth: 0)", + }).map((c) => c.path), + ).toEqual([A_UP, A_DOWN]); + }); +}); diff --git a/cli/test/dbt_module_tracker_unit.test.ts b/cli/test/dbt_module_tracker_unit.test.ts new file mode 100644 index 0000000000..90abee18ae --- /dev/null +++ b/cli/test/dbt_module_tracker_unit.test.ts @@ -0,0 +1,145 @@ +/** + * `buildTracker` decides whose top hash `wmill-lock.yaml` refreshes. A dbt + * project is mostly files that are not Windmill script extensions — the project + * file, `packages.yml`, schema YAML, seed CSVs — and its folder is spelled + * `__dbt\` on Windows, so both the extension gate and a raw-path search for + * `__dbt/` left the descriptor untracked and its hash stale. + */ +import { expect, test, describe, beforeEach, afterEach } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { buildTracker, elementsToMap } from "../src/commands/sync/sync.ts"; +import { isDbtGeneratedPath } from "../src/utils/resource_folders.ts"; +import { readModulesFromDisk } from "../src/commands/script/script.ts"; + +describe("buildTracker with a dbt project", () => { + let dir: string; + let cwd: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "wmill-dbt-tracker-")); + cwd = process.cwd(); + process.chdir(dir); + fs.mkdirSync(path.join(dir, "f/analytics/analytics__dbt/models"), { + recursive: true, + }); + fs.mkdirSync(path.join(dir, "f/analytics/analytics__dbt/seeds"), { + recursive: true, + }); + // The descriptor sits INSIDE the project folder, and `findContentFile` + // resolves the metadata path to it. + fs.writeFileSync(path.join(dir, "f/analytics/analytics.script.yaml"), "{}"); + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__dbt/dbt_project.yml"), + "name: analytics\n", + ); + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__dbt/wm_dbt.yaml"), + "profile: {}\n", + ); + }); + + afterEach(() => { + process.chdir(cwd); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + const tracked = async (p: string) => + (await buildTracker([{ name: "edited", path: p, before: "", after: "" }])) + .scripts; + + test("a model edit selects the descriptor", async () => { + expect( + await tracked("f/analytics/analytics__dbt/models/stg_orders.sql"), + ).toEqual(["f/analytics/analytics__dbt/wm_dbt.yaml"]); + }); + + test("so do the files that are not script extensions", async () => { + for (const p of [ + "f/analytics/analytics__dbt/dbt_project.yml", + "f/analytics/analytics__dbt/packages.yml", + "f/analytics/analytics__dbt/models/_models.yml", + "f/analytics/analytics__dbt/seeds/country_codes.csv", + ]) { + expect(await tracked(p)).toEqual( + ["f/analytics/analytics__dbt/wm_dbt.yaml"], + `${p} left the descriptor untracked`, + ); + } + }); + + // Regression: hoisting the module check above the extension gate made + // `__mod/script.yaml` — a folder-layout script's METADATA, which is an + // entry-point path — look like its own content file. Pushed as one, the + // metadata pass asks for the language of `.yaml` and aborts the command. Not a + // dbt shape at all; reached by editing the summary of any modular script. + test("a modular script's own metadata resolves to its content file", async () => { + fs.mkdirSync(path.join(dir, "f/helpers/util__mod"), { recursive: true }); + fs.writeFileSync(path.join(dir, "f/helpers/util__mod/script.yaml"), "{}"); + fs.writeFileSync( + path.join(dir, "f/helpers/util__mod/script.ts"), + "export function main() {}\n", + ); + expect(await tracked("f/helpers/util__mod/script.yaml")).toEqual([ + "f/helpers/util__mod/script.ts", + ]); + }); + + test("and a Windows-separated path", async () => { + expect( + await tracked("f\\analytics\\analytics__dbt\\models\\stg_orders.sql"), + ).toEqual(["f/analytics/analytics__dbt/wm_dbt.yaml"]); + }); + + // A dbt descriptor is the script's CONTENT and is a `.yaml` inside the + // project folder. `--json` drops every metadata `.yaml` as the twin it does + // not read — dropping this one too leaves a workspace whose dbt scripts have + // metadata, a lock and a project bundle, but nothing to run. + test("--json keeps the descriptor while dropping metadata yaml", async () => { + const file = (path: string) => ({ + path, + isDirectory: false, + getChildren: async function* () {}, + getContentText: async () => "x", + }); + const root = { + path: "", + isDirectory: true, + getChildren: async function* () { + yield file("f/analytics/analytics__dbt/wm_dbt.yaml"); + yield file("f/analytics/analytics.script.yaml"); + yield file("f/analytics/analytics.script.json"); + }, + getContentText: async () => "", + }; + const map = await elementsToMap(root as any, () => false, true, {}); + expect(Object.keys(map).sort()).toEqual([ + "f/analytics/analytics.script.json", + "f/analytics/analytics__dbt/wm_dbt.yaml", + ]); + }); + + // `cp -r my-dbt-project/.` copies whatever the checkout holds, and what a + // `.gitignore` was keeping out of the repo is exactly the file that must not + // become a script version. Both halves: bundled, it is uploaded; offered by + // the diff, every push asks to upload it again. + test("a local .env is neither bundled nor offered as a change", async () => { + const project = path.join(dir, "f/analytics/analytics__dbt"); + fs.writeFileSync(path.join(project, ".env"), "DBT_PASSWORD=hunter2\n"); + fs.writeFileSync(path.join(project, "models/stg.sql"), "select 1"); + + const modules = await readModulesFromDisk(project, undefined, false, true); + // Sorted: the bundle is a set of paths, and the walk follows `readdirSync`, + // whose order is the filesystem's. + expect(Object.keys(modules ?? {}).sort()).toEqual([ + "dbt_project.yml", + "models/stg.sql", + ]); + + // The predicate the sync's ignore filter asks, so the file is not offered + // as an item of its own either. + expect(isDbtGeneratedPath("f/analytics/analytics__dbt/.env")).toBe(true); + expect(isDbtGeneratedPath("f/analytics/analytics__dbt/models/stg.sql")).toBe(false); + }); +}); diff --git a/cli/test/dbt_optional_descriptor_unit.test.ts b/cli/test/dbt_optional_descriptor_unit.test.ts new file mode 100644 index 0000000000..f44bdc6103 --- /dev/null +++ b/cli/test/dbt_optional_descriptor_unit.test.ts @@ -0,0 +1,323 @@ +/** + * An unmodified dbt project is already a complete Windmill script: the + * descriptor is optional, and a project that never names one must push, diff + * and pull without ever growing a Windmill file inside it. + */ +import { expect, test, describe, beforeEach, afterEach } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { FSFSElement, elementsToMap } from "../src/commands/sync/sync.ts"; +import { listWorkspacePaths } from "../src/commands/dev/dev.ts"; +import { + DbtPathCollisionError, + findContentFile, + handleFile, + hasScriptExt, + removeExtensionToPath, +} from "../src/commands/script/script.ts"; +import { pushParentScriptForModule } from "../src/commands/sync/sync.ts"; + +/** The local map's keys are the walk's own — `path.join`, so `__dbt\\` on + * Windows — while a remote's are the API's. The synthesized descriptor follows + * the spelling of the `dbt_project.yml` it was derived from, like every other + * key in that map, so an assertion on one platform's separator tests the + * platform and not the synthesis. */ +const normalized = (m: Record) => + Object.fromEntries( + Object.entries(m).map(([k, v]) => [k.replaceAll("\\", "/"), v]), + ); + +describe("a dbt project without a descriptor", () => { + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "wmill-dbt-nodesc-")); + fs.mkdirSync(path.join(dir, "f/analytics/analytics__dbt/models"), { + recursive: true, + }); + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__dbt/dbt_project.yml"), + "name: analytics\n", + ); + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__dbt/models/stg_orders.sql"), + "select 1", + ); + fs.writeFileSync(path.join(dir, "f/analytics/analytics.script.yaml"), "{}"); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + // Without this the project has no content file, so nothing identifies it as a + // script and the whole project silently never deploys. + test("is still discovered, as an empty descriptor", async () => { + const root = await FSFSElement(dir, [], true); + const map = await elementsToMap(root, () => false, false, {}); + expect(normalized(map)["f/analytics/analytics__dbt/wm_dbt.yaml"]).toBe(""); + }); + + // The metadata has to resolve to a content path that is not on disk, or every + // caller that goes metadata -> content aborts the push. + test("resolves from its metadata to the absent descriptor", async () => { + const cwd = process.cwd(); + process.chdir(dir); + try { + expect(await findContentFile("f/analytics/analytics.script.yaml")).toBe( + "f/analytics/analytics__dbt/wm_dbt.yaml", + ); + } finally { + process.chdir(cwd); + } + }); + + // The descriptor is the one "extension" that contains a separator, so on + // Windows it is spelled `__dbt\wm_dbt.yaml` and a forward-slash suffix test + // matches nothing — every dbt project skipped, with no error. + test("is recognized when the path is spelled with backslashes", () => { + const win = "f\\analytics\\analytics__dbt\\wm_dbt.yaml"; + expect(hasScriptExt(win)).toBe(true); + expect(removeExtensionToPath(win)).toBe("f\\analytics\\analytics"); + }); + + // Both `.py` and `__dbt/` deploy to the SAME remote path, and the + // descriptor is optional — so the project is invisible to the candidate list + // while being perfectly real. Resolved to the ordinary file, a model edit + // deploys the Python script over the dbt one. + test("refuses to resolve when an ordinary script shares its path", async () => { + const cwd = process.cwd(); + process.chdir(dir); + try { + fs.writeFileSync(path.join(dir, "f/analytics/analytics.py"), "def main(): ..."); + const err = await findContentFile("f/analytics/analytics.script.yaml").then( + () => undefined, + (e) => e as Error, + ); + expect(err?.message).toContain("f/analytics/analytics__dbt/dbt_project.yml"); + expect(err?.message).toContain("f/analytics/analytics.py"); + } finally { + process.chdir(cwd); + } + }); + + // The guard has to sit on the push paths themselves, not only on the + // metadata->content resolution: an ordinary file goes straight to + // `handleFile`, and a module edit reaches its parent through a call whose + // errors were swallowed — so each path could still overwrite the other's + // script while reporting success. + test("both push paths refuse the collision", async () => { + const cwd = process.cwd(); + process.chdir(dir); + try { + fs.writeFileSync(path.join(dir, "f/analytics/analytics.py"), "def main(): ..."); + const ordinary = await handleFile( + "f/analytics/analytics.py", + { workspaceId: "w", remote: "http://localhost", name: "w", token: "t" } as any, + [], + undefined, + undefined, + {}, + [], + ).then( + () => undefined, + (e) => e as Error, + ); + expect(ordinary).toBeInstanceOf(DbtPathCollisionError); + expect(ordinary?.message).toContain("f/analytics/analytics.py"); + + const model = await pushParentScriptForModule( + "f/analytics/analytics__dbt/models/stg_orders.sql", + { workspaceId: "w", remote: "http://localhost", name: "w", token: "t" } as any, + [], + undefined, + undefined, + {}, + [], + ).then( + () => undefined, + (e) => e as Error, + ); + expect(model).toBeInstanceOf(DbtPathCollisionError); + } finally { + process.chdir(cwd); + } + }); + + // The guard above must not fire on the project's OWN descriptor: that file is + // the dbt script's content, and its base resolves to the same + // `__dbt/dbt_project.yml` — so a naive check finds the project + // colliding with itself and every dbt push fails before deploying anything. + test("a project does not collide with itself", async () => { + const cwd = process.cwd(); + process.chdir(dir); + const remote = { workspaceId: "w", remote: "http://127.0.0.1:1", name: "w", token: "t" }; + const push = (p: string) => + handleFile(p, remote as any, [], undefined, undefined, {}, []).then( + () => undefined, + (e) => e as Error, + ); + try { + // Descriptor-less: the fixture's project, pushed through the module path. + const nodesc = await pushParentScriptForModule( + "f/analytics/analytics__dbt/models/stg_orders.sql", + remote as any, + [], + undefined, + undefined, + {}, + [], + ).then( + () => undefined, + (e) => e as Error, + ); + expect(nodesc).not.toBeInstanceOf(DbtPathCollisionError); + + // Descriptor present, pushed directly. Both reach the network — which is + // unreachable here on purpose — so anything BUT the collision is a pass. + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__dbt/wm_dbt.yaml"), + "profile: {}\n", + ); + expect(await push("f/analytics/analytics__dbt/wm_dbt.yaml")).not.toBeInstanceOf( + DbtPathCollisionError, + ); + } finally { + process.chdir(cwd); + } + }); + + // The exemption above is only for the project's OWN marker. A descriptor + // pushed DIRECTLY never passes through the metadata resolution that catches + // the collision, so without this it would deploy over the ordinary script + // sitting at the same remote path. + test("but a descriptor still refuses an ordinary script at its path", async () => { + const cwd = process.cwd(); + process.chdir(dir); + try { + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__dbt/wm_dbt.yaml"), + "profile: {}\n", + ); + fs.writeFileSync(path.join(dir, "f/analytics/analytics.py"), "def main(): ..."); + const err = await handleFile( + "f/analytics/analytics__dbt/wm_dbt.yaml", + { workspaceId: "w", remote: "http://127.0.0.1:1", name: "w", token: "t" } as any, + [], + undefined, + undefined, + {}, + [], + ).then( + () => undefined, + (e) => e as Error, + ); + expect(err).toBeInstanceOf(DbtPathCollisionError); + expect(err?.message).toContain("f/analytics/analytics.py"); + } finally { + process.chdir(cwd); + } + }); + + // Both layouts deploy to the same remote path, and the folder layout is the + // one whose base is NOT its filename: `__mod/script.ts` deploys to + // ``, exactly where the dbt project goes. + test("the collision holds for a folder-layout script too", async () => { + const cwd = process.cwd(); + process.chdir(dir); + const remote = { workspaceId: "w", remote: "http://127.0.0.1:1", name: "w", token: "t" }; + const push = (p: string) => + handleFile(p, remote as any, [], undefined, undefined, {}, []).then( + () => undefined, + (e) => e as Error, + ); + try { + fs.mkdirSync(path.join(dir, "f/analytics/analytics__mod"), { recursive: true }); + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__mod/script.ts"), + "export async function main() {}", + ); + fs.writeFileSync( + path.join(dir, "f/analytics/analytics__dbt/wm_dbt.yaml"), + "profile: {}\n", + ); + + // From the dbt side: the descriptor must find the `__mod` entry point. + const fromDbt = await push("f/analytics/analytics__dbt/wm_dbt.yaml"); + expect(fromDbt).toBeInstanceOf(DbtPathCollisionError); + expect(fromDbt?.message).toContain("analytics__mod/script.ts"); + + // And from the ordinary side, whose base is not its filename. + const fromMod = await push("f/analytics/analytics__mod/script.ts"); + expect(fromMod).toBeInstanceOf(DbtPathCollisionError); + expect(fromMod?.message).toContain("analytics__dbt/dbt_project.yml"); + } finally { + process.chdir(cwd); + } + }); + + // The two sides spell "absent" differently — nothing on disk, nothing in the + // export — so without one normalization a descriptor-less project reads as an + // addition on every push and a deletion on every pull, forever. + test("compares equal to a remote that carries no descriptor either", async () => { + const remote = { + path: "", + isDirectory: true, + getChildren: async function* () { + for (const p of [ + "f/analytics/analytics__dbt/dbt_project.yml", + "f/analytics/analytics__dbt/models/stg_orders.sql", + "f/analytics/analytics.script.yaml", + ]) { + yield { + path: p, + isDirectory: false, + getChildren: async function* () {}, + getContentText: async () => "x", + }; + } + }, + getContentText: async () => "", + }; + const local = await elementsToMap( + await FSFSElement(dir, [], true), + () => false, + false, + {}, + ); + const remoteMap = await elementsToMap(remote as any, () => false, false, {}); + const key = "f/analytics/analytics__dbt/wm_dbt.yaml"; + expect(normalized(local)[key]).toBe(""); + expect(normalized(remoteMap)[key]).toBe(""); + }); +}); + +// `wmill dev` walks basenames, and a dbt script is a DIRECTORY whose descriptor +// may not exist — so it is recognized by the project folder or not at all. The +// walk must also stop there: the project's own `.sql` models match the script +// extensions and would each be listed as a script of their own. +describe("dev-mode discovery of a dbt project", () => { + let dir: string + let cwd: string + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'wmill-dbt-dev-')) + fs.mkdirSync(path.join(dir, 'f/analytics/analytics__dbt/models'), { recursive: true }) + fs.writeFileSync(path.join(dir, 'f/analytics/analytics__dbt/dbt_project.yml'), 'name: a\n') + fs.writeFileSync(path.join(dir, 'f/analytics/analytics__dbt/models/stg.sql'), 'select 1') + cwd = process.cwd() + process.chdir(dir) + }) + + afterEach(() => { + process.chdir(cwd) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + test('lists the project itself and nothing inside it', async () => { + const items = await listWorkspacePaths() + const paths = items.map((i) => i.path).sort() + expect(paths).toEqual(['f/analytics/analytics']) + }) +}) diff --git a/cli/test/deploy_on_behalf_of_unit.test.ts b/cli/test/deploy_on_behalf_of_unit.test.ts new file mode 100644 index 0000000000..2214c0dc37 --- /dev/null +++ b/cli/test/deploy_on_behalf_of_unit.test.ts @@ -0,0 +1,83 @@ +import { expect, test } from "bun:test"; +import { deployItem } from "../windmill-utils-internal/src/deploy.ts"; + +// `deployItem` spreads the source item into the request body, and a script's/flow's +// on_behalf_of names a username that only exists in the source +// workspace. Sending it to the target pairs one workspace's principal with the other's +// email, which the backend rejects. Deleting the spread is an easy regression, so pin +// that the key never reaches the wire. +function recordingProvider(captured: [string, any][], flowExists: boolean) { + const source = { + on_behalf_of_email: "alice@corp", + on_behalf_of: "u/alice", + }; + return { + existsFlowByPath: async () => flowExists, + existsScriptByPath: async () => true, + getFlowByPath: async () => ({ + path: "f/x/f", + summary: "", + value: { modules: [] }, + ...source, + }), + createFlow: async (p: any) => void captured.push(["createFlow", p.requestBody]), + updateFlow: async (p: any) => void captured.push(["updateFlow", p.requestBody]), + getScriptByPath: async () => ({ + path: "f/x/s", + summary: "", + content: "x", + language: "bun", + hash: "abc", + ...source, + }), + createScript: async (p: any) => + void captured.push(["createScript", p.requestBody]), + } as any; +} + +test("deployItem: never sends the source workspace's on_behalf_of", async () => { + const captured: [string, any][] = []; + + // The clear is written out once per branch, so exercise all three: a flow that + // does not exist in the target (create), one that does (update — the branch + // `wmill workspace merge` takes for anything already deployed), and a script. + await deployItem( + recordingProvider(captured, false), + "flow" as any, + "f/x/f", + "src", + "dst", + "alice@corp", + ); + await deployItem( + recordingProvider(captured, true), + "flow" as any, + "f/x/f", + "src", + "dst", + "alice@corp", + ); + await deployItem( + recordingProvider(captured, false), + "script" as any, + "f/x/s", + "src", + "dst", + "alice@corp", + ); + + expect(captured.map(([fn]) => fn)).toEqual([ + "createFlow", + "updateFlow", + "createScript", + ]); + for (const [, body] of captured) { + // The email is still overridden with the caller's choice... + expect(body.on_behalf_of_email).toBe("alice@corp"); + expect(body.preserve_on_behalf_of).toBe(true); + // ...while the principal is dropped, so the backend derives the target's own. + expect( + "on_behalf_of" in JSON.parse(JSON.stringify(body)), + ).toBe(false); + } +}); diff --git a/cli/test/dev_recorder_bundle_unit.test.ts b/cli/test/dev_recorder_bundle_unit.test.ts new file mode 100644 index 0000000000..2c7a35f696 --- /dev/null +++ b/cli/test/dev_recorder_bundle_unit.test.ts @@ -0,0 +1,42 @@ +/** + * The recorder `wmill app dev --recording` serves is generated from the + * frontend's raw-app recorder, not written here, so it can silently ship a stale + * event model after that recorder changes. The committed bundle records the + * sources it was built from and their hash; this fails when they no longer + * agree. + * + * Fix a failure with `bun run gen:dev-recorder` from cli/. + */ + +import { describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { hashRecorderSources } from "../generate-dev-recorder.ts"; +import { + DEV_RECORDER_BUNDLE, + DEV_RECORDER_SOURCE_HASH, + DEV_RECORDER_SOURCES, +} from "../src/commands/app/devRecorderBundle.gen.ts"; + +const REPO_ROOT = path.join(import.meta.dir, "..", ".."); + +describe("dev recorder bundle", () => { + test("exposes the recorder factory as a global", () => { + expect(DEV_RECORDER_BUNDLE).toContain("__wmillRecorder"); + expect(DEV_RECORDER_BUNDLE).toContain("createRawAppRecording"); + // Runes are stripped at generation; one left in would throw at load time. + expect(DEV_RECORDER_BUNDLE).not.toContain("$state"); + }); + + test("is up to date with the frontend recorder", () => { + expect(DEV_RECORDER_SOURCES.length).toBeGreaterThan(0); + const present = DEV_RECORDER_SOURCES.every((f) => + fs.existsSync(path.join(REPO_ROOT, f)) + ); + // The published CLI package ships without the frontend sources. + if (!present) return; + expect(hashRecorderSources(DEV_RECORDER_SOURCES, REPO_ROOT)).toBe( + DEV_RECORDER_SOURCE_HASH, + ); + }); +}); diff --git a/cli/test/dev_recorder_routes_unit.test.ts b/cli/test/dev_recorder_routes_unit.test.ts new file mode 100644 index 0000000000..29ad71c983 --- /dev/null +++ b/cli/test/dev_recorder_routes_unit.test.ts @@ -0,0 +1,43 @@ +/** + * Guards on the routes `wmill app dev --recording` adds: what may write a + * recording, what a recording may be named, and that two saves never collide. + */ + +import { expect, test } from "bun:test"; +import { + isOwnOrigin, + isRecordingFileName, + recordingFileName, +} from "../src/commands/app/devRecorder.ts"; + +test("only the shell's own origin may save a recording", () => { + expect(isOwnOrigin("http://localhost:4000", "localhost:4000")).toBe(true); + expect(isOwnOrigin("http://127.0.0.1:4000", "127.0.0.1:4000")).toBe(true); + // A cross-site POST carrying JSON under a simple content type needs no + // preflight, so a foreign origin sharing the port must still be refused. + expect(isOwnOrigin("http://attacker.example:4000", "localhost:4000")).toBe( + false, + ); + expect(isOwnOrigin("null", "localhost:4000")).toBe(false); + // No Origin at all is a non-browser client, not a cross-site page. + expect(isOwnOrigin(undefined, "localhost:4000")).toBe(true); +}); + +test("recording names stay inside the recordings folder", () => { + expect(isRecordingFileName("recording-2026-01-01-00-00-00-000.json")).toBe( + true, + ); + expect(isRecordingFileName("../../../etc/passwd")).toBe(false); + expect(isRecordingFileName("..%2Fx.json")).toBe(false); + expect(isRecordingFileName("sub/dir.json")).toBe(false); + expect(isRecordingFileName("recording.txt")).toBe(false); +}); + +test("two saves in the same millisecond get distinct names", () => { + const now = new Date("2026-01-01T00:00:00.123Z"); + const first = recordingFileName(now, 0); + const second = recordingFileName(now, 1); + expect(first).toBe("recording-2026-01-01-00-00-00-123.json"); + expect(second).not.toBe(first); + expect(isRecordingFileName(second)).toBe(true); +}); diff --git a/cli/test/elements_to_map_branch_specific_unit.test.ts b/cli/test/elements_to_map_branch_specific_unit.test.ts index b9abaa8dd1..ebab7ef929 100644 --- a/cli/test/elements_to_map_branch_specific_unit.test.ts +++ b/cli/test/elements_to_map_branch_specific_unit.test.ts @@ -423,3 +423,45 @@ test("elementsToMap: isRemote undefined behaves like local (backward compatible) // Base file should be skipped (same behavior as isRemote=false) expect(Object.keys(result).includes("f/test.variable.yaml")).toEqual(false); }); + +// ============================================================================= +// REGRESSION TEST: --skip-scripts covers a script's module files +// A module is deployed as part of its parent script, and the module shortcut +// runs before every skip filter — so a changed model under a dbt project's +// `__dbt/` folder pushed the script `--skip-scripts` asked to leave alone. +// ============================================================================= + +test("elementsToMap: script modules are excluded when skipScripts is set", async () => { + const files: MockFile[] = [ + { path: "f/analytics/analytics.dbt.yaml", content: "profile: {}\n" }, + { + path: "f/analytics/analytics__dbt/models/stg_orders.sql", + content: "select 1", + }, + { path: "f/Shared/Variable/TestVar.variable.yaml", content: "value: test" }, + ]; + + const kept = await elementsToMap( + createMockDynFSElement(files), + noIgnore, + false, + defaultSkips, + ); + expect( + Object.keys(kept).includes("f/analytics/analytics__dbt/models/stg_orders.sql"), + ).toEqual(true); + + const skipped = await elementsToMap( + createMockDynFSElement(files), + noIgnore, + false, + { skipScripts: true }, + ); + expect( + Object.keys(skipped).includes("f/analytics/analytics__dbt/models/stg_orders.sql"), + ).toEqual(false); + // Everything else is unaffected. + expect( + Object.keys(skipped).includes("f/Shared/Variable/TestVar.variable.yaml"), + ).toEqual(true); +}); diff --git a/cli/test/git_unit.test.ts b/cli/test/git_unit.test.ts index 9665b8581a..92d44e09d9 100644 --- a/cli/test/git_unit.test.ts +++ b/cli/test/git_unit.test.ts @@ -182,6 +182,30 @@ describe("computeGitSyncDeployBranch", () => { ).toBe("wm_deploy/prod/f__team_a"); }); + test("datatable migration branches off its repo-relative migrations/ path", () => { + const items = [ + { + path_type: "datatable_migration", + path: "migrations/datatable/mydb/20260101000000_add_users", + }, + ]; + expect( + computeGitSyncDeployBranch({ ...base, useIndividualBranch: true, items }) + ).toBe( + "wm_deploy/prod/datatable_migration/migrations__datatable__mydb__20260101000000_add_users" + ); + // group_by_folder collapses every data table's migrations onto one branch — + // the backend's debounce key takes the same two segments. + expect( + computeGitSyncDeployBranch({ + ...base, + useIndividualBranch: true, + groupByFolder: true, + items, + }) + ).toBe("wm_deploy/prod/migrations__datatable"); + }); + test("falls back to parent_path when path is absent", () => { expect( computeGitSyncDeployBranch({ @@ -357,6 +381,17 @@ describe("gitSyncIncludePattern", () => { "f/t.amqp_trigger.*" ); }); + test("datatable migration expands to its two repo-relative .sql files", () => { + expect( + gitSyncIncludePattern( + "datatable_migration", + "migrations/datatable/mydb/20260101000000_add_users" + ) + ).toBe( + "migrations/datatable/mydb/20260101000000_add_users.up.sql," + + "migrations/datatable/mydb/20260101000000_add_users.down.sql" + ); + }); }); describe("deriveGitSyncDeployIncludes", () => { diff --git a/cli/test/gitsync_converter_unit.test.ts b/cli/test/gitsync_converter_unit.test.ts index e84c2ffa11..628b6dd161 100644 --- a/cli/test/gitsync_converter_unit.test.ts +++ b/cli/test/gitsync_converter_unit.test.ts @@ -21,6 +21,24 @@ describe("GitSyncSettingsConverter.fromBackendFormat", () => { expect(result.skipWorkspaceDependencies).toBe(false); }); + test("converts datatablemigration in include_type to skipDatatableMigrations: false", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow", "datatablemigration"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipDatatableMigrations).toBe(false); + }); + + test("sets skipDatatableMigrations: true when datatablemigration is absent", () => { + const backend = { + include_path: ["f/**"], + include_type: ["script", "flow"], + }; + const result = GitSyncSettingsConverter.fromBackendFormat(backend); + expect(result.skipDatatableMigrations).toBe(true); + }); + test("sets skipWorkspaceDependencies: true when workspacedependencies is absent", () => { const backend = { include_path: ["f/**"], @@ -71,6 +89,21 @@ describe("GitSyncSettingsConverter.fromBackendFormat", () => { // ============================================================================= describe("GitSyncSettingsConverter.toBackendFormat", () => { + test("adds datatablemigration when skipDatatableMigrations is false", () => { + expect( + GitSyncSettingsConverter.toBackendFormat({ + includes: ["f/**"], + skipDatatableMigrations: false, + }).include_type, + ).toContain("datatablemigration"); + expect( + GitSyncSettingsConverter.toBackendFormat({ + includes: ["f/**"], + skipDatatableMigrations: true, + }).include_type, + ).not.toContain("datatablemigration"); + }); + test("adds workspacedependencies when skipWorkspaceDependencies is false", () => { const opts = { includes: ["f/**"], diff --git a/cli/test/pipeline_local_graph_unit.test.ts b/cli/test/pipeline_local_graph_unit.test.ts index 7b8b8b0dc5..29d016857c 100644 --- a/cli/test/pipeline_local_graph_unit.test.ts +++ b/cli/test/pipeline_local_graph_unit.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { buildLocalPipelineGraph, + hideDbtRunnables, parseMuteAnnotations, } from "../src/commands/pipeline/localGraph.ts"; @@ -915,3 +916,96 @@ test("pipeline docs renders a `Macro libraries` section (call + `// use`)", asyn }, ); }); + +test("a dbt script is not a pipeline node — a dbt-only folder has no graph", async () => { + await withFolder( + { + "warehouse.dbt.yaml": `engine: dbt-core-1x\nprofile:\n resource: $res:u/admin/wh\n`, + }, + async (root, folder) => { + const { graph, scripts } = await buildLocalPipelineGraph({ + root, + folder, + defaultTs: "bun", + }); + // A dbt project is authored and run as itself, never as a pipeline member; + // the deploy leaves its `auto_kind` unset, so the local graph must agree. + expect(graph.runnables).toEqual([]); + expect(scripts).toEqual([]); + }, + ); +}); + +test("a folder mixing dbt and a pipeline keeps only the pipeline member", async () => { + await withFolder( + { + "warehouse.dbt.yaml": `engine: dbt-core-1x\nprofile:\n resource: $res:u/admin/wh\n`, + "report.bun.ts": `// pipeline\nexport async function main() {}\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ + root, + folder, + defaultTs: "bun", + }); + expect(graph.runnables.map((r) => r.path)).toEqual(["f/mypipe/report"]); + }, + ); +}); + +test("hideDbtRunnables drops the dbt node from a deployed graph, keeping its relations", () => { + // `/assets/graph` is asset-usage driven, so it returns the dbt script as a + // producer. The CLI's deployed views must not render it as a pipeline script. + const deployed = { + runnables: [ + { path: "f/x/dbtproj", usage_kind: "script" as const, dbt: { model_count: 2 } }, + { path: "f/x/report", usage_kind: "script" as const, in_pipeline: true }, + ], + assets: [ + { kind: "table", path: "u/a/wh/s/stg_orders" }, + { kind: "table", path: "u/a/wh/s/fct_orders" }, + ], + edges: [ + { runnable_kind: "script", runnable_path: "f/x/dbtproj", asset_kind: "table", asset_path: "u/a/wh/s/stg_orders", access_type: "w" as const }, + { runnable_kind: "script", runnable_path: "f/x/dbtproj", asset_kind: "table", asset_path: "u/a/wh/s/fct_orders", access_type: "w" as const }, + { runnable_kind: "script", runnable_path: "f/x/report", asset_kind: "table", asset_path: "u/a/wh/s/fct_orders", access_type: "r" as const }, + ], + triggers: [ + { trigger_kind: "asset" as const, asset_kind: "table", asset_path: "u/a/wh/s/fct_orders", runnable_kind: "script", runnable_path: "f/x/report" }, + ], + }; + const g = hideDbtRunnables(deployed); + expect(g.runnables.map((r) => r.path)).toEqual(["f/x/report"]); + expect(g.edges.map((e) => e.runnable_path)).toEqual(["f/x/report"]); + // The relations stay: they are what the downstream pipeline script reads. + expect(g.assets).toHaveLength(2); + expect(g.triggers).toHaveLength(1); +}); + +test("hideDbtRunnables keeps a flow sharing a path with the dbt script", () => { + // Runnable identity in the graph is `(usage_kind, path)`; a script and a flow + // may share a path, so only the dbt script may be removed. + const g = hideDbtRunnables({ + runnables: [ + { path: "f/x/proj", usage_kind: "script", dbt: { model_count: 1 } }, + { path: "f/x/proj", usage_kind: "flow" }, + ], + edges: [ + { runnable_kind: "flow", runnable_path: "f/x/proj" }, + { runnable_kind: "script", runnable_path: "f/x/proj" }, + ], + triggers: [{ runnable_kind: "flow", runnable_path: "f/x/proj" }], + }); + expect(g.runnables.map((r) => r.usage_kind)).toEqual(["flow"]); + expect(g.edges.map((e) => e.runnable_kind)).toEqual(["flow"]); + expect(g.triggers).toHaveLength(1); +}); + +test("hideDbtRunnables is a no-op when the folder has no dbt project", () => { + const g = { + runnables: [{ path: "f/x/a", usage_kind: "script" }], + edges: [], + triggers: [], + }; + expect(hideDbtRunnables(g)).toBe(g); +}); diff --git a/cli/test/raw_app_recordings_skip_unit.test.ts b/cli/test/raw_app_recordings_skip_unit.test.ts new file mode 100644 index 0000000000..408671cfe3 --- /dev/null +++ b/cli/test/raw_app_recordings_skip_unit.test.ts @@ -0,0 +1,63 @@ +/** + * `wmill app dev --recording` writes multi-MB session recordings into + * `.raw_app/recordings/`. They are local artifacts: the sync differ must + * not offer them as app source (the push itself drops them in + * `collectAppFiles`, so a differ that still sees them reports a change that + * pushing can never settle). + */ + +import { expect, test } from "bun:test"; +import { sep as SEP } from "node:path"; +import { elementsToMap } from "../src/commands/sync/sync.ts"; + +type MockFile = { path: string; content: string }; + +// FSFSElement joins with the platform separator, and the exclusion has to hold +// on Windows too. +const p = (...parts: string[]) => parts.join(SEP); + +function mockElement(files: MockFile[]) { + return { + isDirectory: true, + path: "", + async getContentText() { + return ""; + }, + async *getChildren() { + for (const file of files) { + yield { + isDirectory: false, + path: file.path, + async getContentText() { + return file.content; + }, + async *getChildren() {}, + }; + } + }, + }; +} + +test("elementsToMap skips recordings/ at the root of a raw app folder only", async () => { + const app = p("f", "demo", "myapp.raw_app"); + const files: MockFile[] = [ + { path: p(app, "index.tsx"), content: "export {}" }, + { + path: p(app, "recordings", "recording-2026-01-01-00-00-00.json"), + content: '{"version":1}', + }, + // The dev server never writes here, so this is the app's own source. + { path: p(app, "src", "recordings", "fixture.json"), content: "{}" }, + ]; + + const result = await elementsToMap( + mockElement(files) as any, + () => false, + false, + {}, + ); + + expect(Object.keys(result).sort()).toEqual( + [p(app, "index.tsx"), p(app, "src", "recordings", "fixture.json")].sort(), + ); +}); diff --git a/cli/test/raw_app_workspace_deps_unit.test.ts b/cli/test/raw_app_workspace_deps_unit.test.ts new file mode 100644 index 0000000000..a76064e8b9 --- /dev/null +++ b/cli/test/raw_app_workspace_deps_unit.test.ts @@ -0,0 +1,84 @@ +/** + * Raw app workspace dependencies + * + * A raw app keeps its runnables in `backend/`, not in `raw_app.yaml`. The + * workspace dependency filtering must resolve those files, otherwise the + * default `dependencies/package.json` is dropped and locks are regenerated + * against unpinned versions. + * + * Exercised through the legacy (tree-less) path: tree mode sources its deps + * from `getMismatchedWorkspaceDeps()`, which is only populated by an + * `uploadScripts` round-trip, so it cannot run offline. Both paths filter the + * same `appValue`, so resolving it correctly is what this pins. + */ + +import { expect, test } from "bun:test"; +import * as path from "node:path"; +import os from "node:os"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { generateAppLocksInternal } from "../src/commands/app/app_metadata.ts"; +import { Workspace } from "../src/commands/workspace/workspace.ts"; + +const stubWorkspace: Workspace = { + remote: "http://localhost:0/", + workspaceId: "test", + name: "test", + token: "test", +}; + +const APP_FOLDER = path.join("f", "example.raw_app"); + +async function withTempDir(fn: (tempDir: string) => Promise): Promise { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "wmill_raw_app_deps_")); + const originalCwd = process.cwd(); + try { + process.chdir(tempDir); + await fn(tempDir); + } finally { + process.chdir(originalCwd); + await rm(tempDir, { recursive: true, force: true }); + } +} + +test("raw app: default workspace deps are picked up from backend runnables", async () => { + await withTempDir(async () => { + await mkdir(path.join(APP_FOLDER, "backend"), { recursive: true }); + await writeFile( + path.join(APP_FOLDER, "raw_app.yaml"), + `summary: "example raw app"\npolicy:\n execution_mode: publisher\n triggerables: {}\n`, + "utf-8", + ); + await writeFile( + path.join(APP_FOLDER, "backend", "test.ts"), + `import * as wmill from "windmill-client"\n\nexport async function main() {\n return wmill.getVariable("example")\n}\n`, + "utf-8", + ); + + await generateAppLocksInternal( + APP_FOLDER, + true, + false, + stubWorkspace, + { defaultTs: "bun" }, + true, // justUpdateMetadataLock — no backend round-trip + true, + ); + + expect( + await generateAppLocksInternal(APP_FOLDER, true, true, stubWorkspace, { defaultTs: "bun" }, false, true), + ).toBeUndefined(); + + // The runnable has no `package_json` annotation, so it uses the default + // manifest — adding it must invalidate the app. + await mkdir("dependencies", { recursive: true }); + await writeFile( + path.join("dependencies", "package.json"), + `{"dependencies": {"windmill-client": "1.742.0"}}`, + "utf-8", + ); + + expect( + await generateAppLocksInternal(APP_FOLDER, true, true, stubWorkspace, { defaultTs: "bun" }, false, true), + ).toEqual("f/example.raw_app"); + }); +}); diff --git a/cli/test/resource_folders_unit.test.ts b/cli/test/resource_folders_unit.test.ts index 0d07812124..de3daa18cd 100644 --- a/cli/test/resource_folders_unit.test.ts +++ b/cli/test/resource_folders_unit.test.ts @@ -3,7 +3,7 @@ * Tests both dotted (.flow, .app, .raw_app) and non-dotted (__flow, __app, __raw_app) modes. */ -import { expect, test, describe, beforeEach } from "bun:test"; +import { expect, test, describe, beforeEach, afterEach } from "bun:test"; import { setNonDottedPaths, getNonDottedPaths, @@ -37,6 +37,13 @@ import { transformJsonPathToDir, isModuleEntryPoint, getScriptBasePathFromModulePath, + isDbtGeneratedPath, + dbtGeneratedDirs, + isUnderGeneratedDir, + isDbtModulePath, + isBundledModuleFile, + moduleFileExclusion, + MAX_MODULE_BYTES, } from "../src/utils/resource_folders.ts"; import { removeWorkerPrefix } from "../src/commands/worker-groups/worker-groups.ts"; @@ -657,3 +664,194 @@ describe("removeWorkerPrefix", () => { expect(removeWorkerPrefix("worker__")).toBe(""); }); }); + +// A dbt project's generated directories never belong to the bundle: hashing and +// uploading a local `target/` would make every local `dbt run` look like a +// project change, and a stale manifest in it is what the runtime reads as the +// graph. They are configurable, so they are read from the project. +describe("dbtGeneratedDirs", () => { + const fs = require("node:fs"); + const os = require("node:os"); + const nodePath = require("node:path"); + let dir: string; + // Every temp directory this block makes, so none outlives the run: the helper + // below mints one per call by design. + let made: string[] = []; + + beforeEach(() => { + dir = fs.mkdtempSync(nodePath.join(os.tmpdir(), "dbtgen-")); + made = [dir]; + }); + + afterEach(() => { + for (const d of made) fs.rmSync(d, { recursive: true, force: true }); + }); + + // A fresh directory per call: `dbtGeneratedDirs` memoizes per project folder, + // since within one sync the project file does not change under it. + const write = (yml: string) => { + const d = fs.mkdtempSync(nodePath.join(os.tmpdir(), "dbtgen-")); + made.push(d); + fs.writeFileSync(nodePath.join(d, "dbt_project.yml"), yml); + return dbtGeneratedDirs(d); + }; + + test("defaults apply with no project file", () => { + const dirs = dbtGeneratedDirs(dir); + expect(dirs.has("target")).toBe(true); + expect(dirs.has("dbt_packages")).toBe(true); + }); + + test("reads nested target-path and packages-install-path", () => { + const dirs = write( + 'name: p\ntarget-path: "build/target"\npackages-install-path: ./vendor/pkgs\n', + ); + expect(dirs.has("build/target")).toBe(true); + expect(dirs.has("vendor/pkgs")).toBe(true); + }); + + test("reads clean-targets in both of dbt's spellings", () => { + expect(write('name: p\nclean-targets: ["a", b]\n').has("a")).toBe(true); + const block = write("name: p\nclean-targets:\n - out/one\n - two\nmodels: {}\n"); + expect(block.has("out/one")).toBe(true); + expect(block.has("two")).toBe(true); + expect(block.has("models")).toBe(false); + }); + + test("ignores a configured path that escapes the project", () => { + const dirs = write('name: p\ntarget-path: "../../etc"\n'); + expect([...dirs].some((d) => d.includes(".."))).toBe(false); + }); +}); + +describe("isUnderGeneratedDir", () => { + const dirs = new Set(["target", "build/target"]); + + test("matches the directory and everything under it", () => { + expect(isUnderGeneratedDir("target", dirs)).toBe(true); + expect(isUnderGeneratedDir("target/manifest.json", dirs)).toBe(true); + expect(isUnderGeneratedDir("build/target/run_results.json", dirs)).toBe(true); + }); + + test("does not match a sibling sharing the prefix", () => { + expect(isUnderGeneratedDir("targetx/a.sql", dirs)).toBe(false); + expect(isUnderGeneratedDir("models/target_helper.sql", dirs)).toBe(false); + expect(isUnderGeneratedDir("build/targeted/a.sql", dirs)).toBe(false); + }); +}); + +// A Windows path spells the folder `__dbt\\`. A lookup that searched the raw +// path for `__dbt/` would find nothing, so a module-only edit would return +// without deploying its parent while the file was still recorded as synced. +describe("dbt module paths on either separator", () => { + test("isDbtModulePath and getScriptBasePathFromModulePath accept backslashes", () => { + expect(isDbtModulePath("f\\x\\proj__dbt\\models\\a.sql")).toBe(true); + expect(getScriptBasePathFromModulePath("f\\x\\proj__dbt\\models\\a.sql")).toBe( + "f/x/proj", + ); + expect(getScriptBasePathFromModulePath("f/x/proj__dbt/models/a.sql")).toBe( + "f/x/proj", + ); + }); + + test("a path with no module folder has no base path", () => { + expect(getScriptBasePathFromModulePath("f/x/proj.dbt.yaml")).toBeUndefined(); + }); +}); + +// The push, the staleness hash and the sync diff all ask this question. They +// must agree: a file one drops and another keeps is a change no push resolves. +describe("isBundledModuleFile", () => { + const fs = require("node:fs"); + const os = require("node:os"); + const nodePath = require("node:path"); + let dir: string; + + beforeEach(() => { + dir = fs.mkdtempSync(nodePath.join(os.tmpdir(), "bundled-")); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + const write = (name: string, data: Buffer | string) => { + const p = nodePath.join(dir, name); + fs.writeFileSync(p, data); + return p; + }; + + test("keeps text, including empty and unicode", () => { + expect(isBundledModuleFile(write("a.sql", "select 1\n"))).toBe(true); + expect(isBundledModuleFile(write("empty.sql", ""))).toBe(true); + expect(isBundledModuleFile(write("u.sql", "select 'café'\n"))).toBe(true); + }); + + test("drops a binary file, which a NUL identifies", () => { + expect(isBundledModuleFile(write("x.png", Buffer.from([0x89, 0x50, 0x00, 0x1a])))).toBe( + false, + ); + }); + + test("drops a file over the per-file limit", () => { + expect(isBundledModuleFile(write("huge.csv", "x".repeat(MAX_MODULE_BYTES + 1)))).toBe( + false, + ); + }); + + // The two reasons a file is not carried are not interchangeable: sync hides + // dbt's binary leftovers, but an oversized seed the project authored has to + // stay in the diff, or the push that reports the size error never runs and the + // remote project is left incomplete without saying so. + test("says WHY a file is not carried", () => { + expect(moduleFileExclusion(write("a.sql", "select 1\n"))).toBe(undefined); + expect(moduleFileExclusion(write("x.png", Buffer.from([0x89, 0x50, 0x00, 0x1a])))).toBe( + "binary", + ); + expect(moduleFileExclusion(write("seed.csv", "x".repeat(MAX_MODULE_BYTES + 1)))).toBe( + "oversized", + ); + }); + + // A pull asks this about files that do not exist locally yet. Answering "not + // carried" there made sync ignore the whole incoming project and write + // nothing — the bundle silently vanished on every fresh checkout. + test("treats a missing file as carried, not as excluded", () => { + expect(isBundledModuleFile(nodePath.join(dir, "does-not-exist.sql"))).toBe(true); + }); +}); + +test("a nested __mod inside a dbt project does not steal the script boundary", () => { + // dbt owns its directory names verbatim, so a folder ending `__mod` is legal + // inside a project. The script is the OUTER module boundary. + expect( + getScriptBasePathFromModulePath("f/x/proj__dbt/models/legacy__mod/a.sql"), + ).toBe("f/x/proj"); + expect(getScriptBasePathFromModulePath("f/x/proj__dbt/models/a.sql")).toBe( + "f/x/proj", + ); + expect(getScriptBasePathFromModulePath("f/x/s__mod/inner.ts")).toBe("f/x/s"); +}); + +test("a nested __mod inside a dbt project is not a module entry point", () => { + expect(isModuleEntryPoint("f/x/s__mod/script.ts")).toBe(true); + // `legacy__mod` is a legal dbt directory; its script.ts belongs to the dbt + // project, not to a module folder of its own. + expect(isModuleEntryPoint("f/x/proj__dbt/models/legacy__mod/script.ts")).toBe( + false, + ); + expect(isModuleEntryPoint("f/x/proj__dbt/models/a.sql")).toBe(false); +}); + +test("a __dbt directory nested inside an ordinary module is not a dbt project file", () => { + expect(isDbtModulePath("f/x/proj__dbt/models/a.sql")).toBe(true); + // `vendor/x__dbt/` belongs to the `foo__mod` script, not to a dbt project. + expect(isDbtModulePath("f/x/foo__mod/vendor/x__dbt/a.ts")).toBe(false); + expect(isDbtModulePath("f/x/foo__mod/helper.ts")).toBe(false); +}); + +test("a __dbt/target nested in an ordinary module is not generated dbt output", () => { + expect(isDbtGeneratedPath("f/x/proj__dbt/target/manifest.json")).toBe(true); + // Belongs to the `foo__mod` script; excluding it would drop a real edit. + expect(isDbtGeneratedPath("f/x/foo__mod/vendor/x__dbt/target/a.ts")).toBe(false); +}); diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index 42874fda97..047d50ee78 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -460,6 +460,10 @@ export async function deployItem( ...flow, preserve_on_behalf_of: preserveOnBehalfOf, on_behalf_of_email: onBehalfOf, + // Usernames are per-workspace, so the source's principal names nobody in + // the target (or names a different person). Clearing it lets the backend + // derive the target's own principal from the email above. + on_behalf_of: undefined, }, }); } else { @@ -469,6 +473,10 @@ export async function deployItem( ...flow, preserve_on_behalf_of: preserveOnBehalfOf, on_behalf_of_email: onBehalfOf, + // Usernames are per-workspace, so the source's principal names nobody in + // the target (or names a different person). Clearing it lets the backend + // derive the target's own principal from the email above. + on_behalf_of: undefined, }, }); } @@ -493,6 +501,8 @@ export async function deployItem( parent_hash: parentHash, preserve_on_behalf_of: preserveOnBehalfOf, on_behalf_of_email: onBehalfOf, + // See the flow branch: a source-workspace principal is never valid here. + on_behalf_of: undefined, }, }); } else if (kind === "app" || kind === "raw_app") { diff --git a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts index 94d9085b94..6aaeea4db1 100644 --- a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts +++ b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts @@ -36,6 +36,7 @@ export const LANGUAGE_EXTENSIONS: Record = { bunnative: "ts", ruby: "rb", rlang: "r", + dbt: "dbt.yaml", // for related places search: ADD_NEW_LANG }; @@ -84,6 +85,7 @@ export const EXTENSION_TO_LANGUAGE: Record = { "playbook.yml": "ansible", "java": "java", "duckdb.sql": "duckdb", + "dbt.yaml": "dbt", "rb": "ruby", // Plain .ts defaults to bun (will be overridden by defaultTs setting) "ts": "bun", diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 6e3f218fd6..b827208c1d 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -45,6 +45,19 @@ RUN apt-get install -y ruby ruby-bundler RUN apt-get install -y r-base-dev \ && Rscript -e 'install.packages("renv", lib="/usr/lib/R/library", repos="https://cloud.r-project.org")' +# dbt +# NO dbt engine is baked in. Fusion may not be: its license grants only a +# non-transferable, non-sublicensable redistribution right. dbt-core 1.x cannot +# be, because its adapter is a Python package chosen per project. dbt-core 2.x +# could be — one adapter-agnostic binary — but shipping a pre-release nobody is +# defaulted onto costs a layer in every image and a version pinned in two places +# that nothing keeps in step. The worker fetches whichever engine a project asks +# for on first use and caches it (docs/dbt-runtime.md, decision 1). +# +# An operator who wants one pre-staged — an air-gapped instance, or a fleet that +# should not fetch per worker — populates `DBT_BUNDLED_DIR` (default +# /usr/local/dbt) with `core2x-/dbt-sa-cli` in their own image layer. + # Fix UV cache permissions for non-root user support (uid 1000, etc.) # The uv tool install ansible command populates the UV cache with root-owned files RUN chmod -R a+rw /tmp/windmill/cache/uv && \ diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 14bfceca50..5b5347558a 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -78,6 +78,19 @@ RUN apt-get install -y iptables # Kerberos runtime RUN apt-get install -y libsasl2-modules-gssapi-mit krb5-user +# dbt +# NO dbt engine is baked in. Fusion may not be: its license grants only a +# non-transferable, non-sublicensable redistribution right. dbt-core 1.x cannot +# be, because its adapter is a Python package chosen per project. dbt-core 2.x +# could be — one adapter-agnostic binary — but shipping a pre-release nobody is +# defaulted onto costs a layer in every image and a version pinned in two places +# that nothing keeps in step. The worker fetches whichever engine a project asks +# for on first use and caches it (docs/dbt-runtime.md, decision 1). +# +# An operator who wants one pre-staged — an air-gapped instance, or a fleet that +# should not fetch per worker — populates `DBT_BUNDLED_DIR` (default +# /usr/local/dbt) with `core2x-/dbt-sa-cli` in their own image layer. + # Fix UV cache permissions for non-root user support (uid 1000, etc.) # The uv tool install ansible command populates the UV cache with root-owned files RUN chmod -R a+rw /tmp/windmill/cache/uv && \ diff --git a/docs/agent-worker-e2e.md b/docs/agent-worker-e2e.md new file mode 100644 index 0000000000..e1fd122b98 --- /dev/null +++ b/docs/agent-worker-e2e.md @@ -0,0 +1,131 @@ +# Running an agent worker locally, for e2e + +An agent worker reaches the database only through the API, so whole code paths +(`Connection::Http`) are never taken by a normal `cargo run`. Exercising them +needs a real one. Every step below has a failure mode that looks like something +else; they are listed with the error each produces. + +## 1. Build with the right features + +Four features, and the agent's own mode gate is the one that is easy to miss: + +```bash +cd backend +cargo build --features quickjs,private,enterprise,license,agent_worker_server +``` + +- `agent_worker_server` mounts `/api/agent_workers/*` on the SERVER. Without it, + `create_agent_token` returns **404** with an empty body. +- `enterprise` + `license` compile the agent MODE into the binary. Without them + the worker exits immediately with `Agent mode is only available in the EE`, + even though the server side works and mints tokens happily. + +Verify before spending time on the handshake — the panic string must be absent: + +```bash +strings target/debug/windmill | grep -c "Agent mode is only available in the EE" # want 0 +``` + +**Pin this feature set for the whole session.** `target/debug/windmill` is one +path shared by every feature combination, and cargo swaps the cached artifact in +and out as the set changes — a `cargo build --features quickjs` (or any build +with a different set) in another pane silently replaces the binary the server and +agent are about to run, and the swap back "completes" in under a second, so it +does not look like a rebuild happened. The symptom is the agent 401ing again +after it had been working, or the EE panic reappearing. Re-run the `strings` +check above whenever anything unexpected regresses, and start the server and the +agent from the SAME build. + +## 2. Run the server without a local worker + +`MODE=server` so nothing else drains the queue and the agent is provably the one +that ran the job: + +```bash +DATABASE_URL= PORT=8420 MODE=server ./target/debug/windmill +``` + +## 3. Mint a token — with an expiry, and unquoted + +```bash +TOK=$(curl -s -X POST localhost:8420/api/auth/login -H 'Content-Type: application/json' \ + -d '{"email":"admin@windmill.dev","password":"changeme"}') + +AT=$(curl -s -X POST localhost:8420/api/agent_workers/create_agent_token \ + -H "Authorization: Bearer $TOK" -H 'Content-Type: application/json' \ + -d '{"worker_group":"agentgrp","tags":["dbt"],"exp":1900000000}' | tr -d '"') +``` + +Two traps, both of which surface as a bare `401` on the agent and a decoded +reason only in the SERVER log: + +- **`exp` must be a real timestamp.** `"exp": null` mints a token the validator + rejects with `Missing required claim: exp`. +- **The response is JSON, so it arrives quoted.** Keeping the `"` gives + `Base64 error: Invalid byte 34, offset 0` — hence the `tr -d '"'`. + +Pass the token **exactly as minted**. It looks like `jwt_agent_` and the +client appends its own hostname-derived suffix to form `jwt_agent__`, +which is what the server splits on. Adding a suffix yourself yields +`Base64 error: Encoded text cannot have a 6-bit remainder`. + +## 4. Start the agent + +`WORKER_TAGS` must contain the tag the JOBS carry, not a tag you invent — a job +whose tag nothing serves sits in `v2_job_queue` forever and looks like a hang. +dbt scripts default to the `dbt` tag. + +```bash +AGENT_TOKEN="$AT" BASE_INTERNAL_URL=http://localhost:8420 MODE=agent \ + WINDMILL_DIR=/home/$USER/wmagent \ + WORKER_GROUP=agentgrp WORKER_TAGS=dbt PORT=8499 ./target/debug/windmill +``` + +`WINDMILL_DIR` off `/tmp` matters on a dev box. Jobs fail with `IoErr: Disk quota +exceeded (os error 122)` while writing the project's files, and `df` looks +healthy — free space and free inodes both. `/tmp` is a tmpfs and Linux supports +per-user quotas on it, so the limit is the user's, not the filesystem's; several +agent sessions' caches under `/tmp` are enough to reach it. Point the worker at a +real disk instead of trying to clean up under the quota. + +Confirm it registered rather than trusting a quiet log: + +```sql +SELECT worker FROM worker_ping + WHERE worker_group = 'agentgrp' AND ping_at > now() - interval '2 min'; +-- ag-agentgrp-- +``` + +## Reading failures + +The agent only ever prints `Agent worker cannot connect to server. Please check +AGENT_TOKEN and BASE_INTERNAL_URL`. The actual reason is in the server log, from +`windmill-api-agent-workers/src/ee.rs` — grep it for `JWT_AGENT auth error`. + +## Confirming the agent is what ran the job + +`worker` on the completed job starts with `ag-`: + +```bash +curl -s -H "Authorization: Bearer $TOK" \ + "localhost:8420/api/w//jobs_u/completed/get/" | jq -r .worker +``` + +## What dbt does on an agent worker + +Runs, retries, and publishes its graph — including a per-run snapshot for a +dynamic descriptor, which it POSTs to `/api/agent_workers/dbt_graph/{workspace}` +rather than writing itself. + +What it does not get is LIVE progress: the reporter tails a JSON event log and +needs a SQL connection, so per-model state is settled from `run_results.json` +when the run ends. Retry state lives only in the worker-local generation, since +there is no database row to arbitrate against — which is why `state_dir` is keyed +by principal. + +Confirming a run really exercised that path: + +```sql +SELECT job_id, count(*) FROM dbt_node WHERE script_path = '' GROUP BY job_id; +-- a row keyed to the JOB id (not the zero UUID) means the agent published a snapshot +``` diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md new file mode 100644 index 0000000000..62e680d3a1 --- /dev/null +++ b/docs/dbt-runtime.md @@ -0,0 +1,1165 @@ +# Windmill as a dbt runtime + +Implementation spec for running an existing dbt project on Windmill with no +changes to the project itself. Companion to [`pipelines-vs-dbt.md`](./pipelines-vs-dbt.md), +which covers the opposite direction (native pipeline features that replace dbt). +The two are complementary: this is the adoption ramp, that is the long game. + +Benchmark to beat is Airflow + [astronomer-cosmos](https://astronomer.github.io/astronomer-cosmos/), +the dominant way dbt is orchestrated today. + +## Scope + +- **In**: run an unmodified dbt project synced into Windmill, one Windmill job per + invocation, live per-model observability, dbt models as first-class assets in + the existing asset graph. +- **Out**: one Windmill job per dbt model, `state:modified` / slim CI, + `dbt docs` hosting, semantic layer, dbt platform integration. +- **CE**: the runtime, the manifest ingest, the asset graph and every piece of + UI ship in CE, as do all adapters except two. Only the `mssql` and `oracle` + adapters are EE, mirroring the native `ScriptLang` boundary (decision 21). + +## Decision log + +| # | Decision | Resolution | +|---|---|---| +| 1 | dbt engine | Three-way toggle (`dbt-core-1x` \| `dbt-core-2x` \| `fusion`); shipped default `dbt-core-1x`, instance-configurable. See below | +| 2 | Artifact shape | `ScriptLang::Dbt` | +| 3 | Graph in v0 | Yes, both runtime and graph | +| 4 | Execution granularity | One job per invocation | +| 5 | Project storage | The project is the script's module bundle; nothing is cloned. See "Where the dbt project lives" | +| 6 | Multiple run configs | Per-run `select` on one script; N scripts means N projects | +| 7 | Run-time `select` | Descriptor default plus run-arg override | +| 8 | Credentials | Workspace warehouses, plus `profiles.yml` passthrough. A descriptor never names a resource. See below | +| 9 | Adapter mappings | postgres, redshift, mysql, snowflake, bigquery, databricks; others via the project's own `profiles.yml` | +| 10 | Private repo auth | Not applicable: the project is synced, not fetched | +| 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | +| 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | +| 13 | Manifest storage | Sidecar table for nodes/edges. Full manifest **not** stored — see below | +| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below | +| 15 | Node rendering | Asset nodes per model plus one runnable node for the script | +| 16 | Progress | Live, from the JSON event stream | +| 17 | Test failures | Honor dbt's own `severity` | +| 18 | Retry | Automatic node-level retry in-job, plus `dbt retry` as a run argument. See below | +| 19 | Caching | Worker-local global cache, keyed by the project digest and the resolution the deploy pinned | +| 20 | Images | Full images only | +| 21 | Licensing | CE except the `mssql` / `oracle` adapters. See below | +| 22 | Naming | Match Cosmos field names; importer deferred | +| 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | +| 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | + +## Decision 1: engine toggle, and why the shipped default is not Fusion yet + +`engine: dbt-core-1x | dbt-core-2x | fusion` in the descriptor. Omitted, it is +`dbt-core-1x`, which runs today's projects untouched. + +No engine is baked into any image. Each is fetched or built on first use and +cached, for a different reason in each case. + +| Engine | Distribution | Cold start | License | +|---|---|---|---| +| `dbt-core-1x` (default) | A uv venv resolved per adapter on first use, then cached. **Cannot** be baked: the adapter is a Python package chosen per project | One venv build per (core range, adapter) | Apache 2.0 | +| `dbt-core-2x` | One adapter-agnostic Rust binary, fetched from GitHub releases on first use, cached | One download | Apache 2.0 | +| `fusion` | **Never bundled.** Fetched from dbt Labs on first use, cached | One download (~290MB) | dbt Fusion engine license agreement | + +2.x is the one that *could* be baked, and deliberately is not: it is a +pre-release (`2.0.0-alpha.5`) that nothing is defaulted onto, so baking it costs +a layer in every image and a version pinned in two places with nothing keeping +them in step. An operator who wants an engine pre-staged — an air-gapped +instance, or a fleet that should not fetch per worker — populates +`DBT_BUNDLED_DIR` (default `/usr/local/dbt`) with `core2x-/dbt-sa-cli` +in a derived image; the worker prefers it over its own cache. + +Two things to know before choosing 2.x: it is a pre-release, and it does not +emit the per-node events the run page animates (see "Live per-model progress"), +so a run on it reports its models only at the end. + +The 1.x venv resolves `dbt-core>=1.8,<2.0.0` *together with* the adapter rather +than pinning a core version, because several adapters cap below the newest core +(dbt-oracle and dbt-databricks below 1.12) and an independent pin makes those +projects unprovisionable. The lockfile records whichever version the resolver +actually chose. Both bounds and each engine version are env-overridable +(`DBT_CORE_1X_FLOOR`, `DBT_CORE_1X_CEILING`, `DBT_CORE_2X_VERSION`). + +Fusion is the fastest option and the toggle exists so users can choose it. Two +things block making it the *shipped* default, both verifiable rather than matters +of taste: + +1. **Redistribution terms.** The Fusion license grants only a "limited, + non-exclusive, non-transferable, non-sublicensable" redistribution right, and + 4.1 forbids introducing "obstacles or delays that have the effect of hampering + or interfering with (a) communication between Provider and End User, (b) + User's ability to view, access, or use the Product and/or any Account + Features." A sandboxed non-interactive job runner sits squarely in that + clause's path, and "may not share, pool, or relay its own login credentials to + any End User" reads directly onto putting one dbt platform token in a + workspace secret. That needs counsel, not an engineering judgment. + **Fetch-at-runtime is the mitigation**: the user's own instance pulls the + binary from dbt Labs directly, so Windmill never redistributes and never + interposes. Do not bake Fusion into any image. +2. **Fusion is v2 semantics, and v2 drops all deprecated functionality.** Every + deprecation warning, including historic ones and those added in 1.10, must be + resolved before a project runs on it. An arbitrary existing dbt 1.x project + therefore may not run unchanged, which is this feature's entire premise. dbt + ships an autofix tool and Fusion/Core interoperate side by side, so it is a + migration users can do, but not one Windmill should silently require of them. + +Consequence: ship with `dbt-core-1x`, which runs today's projects untouched, and +flip the instance default to `fusion` once counsel clears the runtime-fetch model +and a real project is verified end to end on it. Both dbt-core engines are +exercised by the e2e suite, so the flip is a config change, not a port. + +## Decision 21: mirror the native warehouse boundary, do not invent one + +Everything structural is CE: the executor, all three engines, the manifest +ingest, the `dbt://` asset graph, live progress, the editor. The only gate is +on two adapters, and it is not a dbt-specific policy — it is the same boundary +the native script languages already draw. Since `bigquery` and `snowflake` +became CE, the only warehouse `ScriptLang`s still behind a license are `mssql` +and `oracledb`, so those two dbt adapters are EE and every other one (postgres, +mysql, duckdb, snowflake, bigquery, databricks, redshift, clickhouse, +salesforce) is CE. Gating any of the others would make reaching a warehouse +through dbt stricter than reaching it natively, which is backwards. + +Those two are *recognized* (for the gate and for the pip package the 1.x +engine's venv needs), not rendered from a resource: an `oracledb` resource is +`{user, password, database}` with no host/protocol/service, and dbt-sqlserver +needs an ODBC `driver` the images do not install. Both reach their warehouse +through the project's own `profiles.yml`, which is also how duckdb, clickhouse +and salesforce work. + +The gate almost never fires in practice: `dbt-core-2x` supports neither adapter, +so it can only apply to `dbt-core-1x` with one of those two. + +**The mechanism differs from the native languages.** They gate at compile time, +so a CE binary simply lacks the executor. That is not available here: there is +one dbt executor and the adapter is only known once the profile resolves. So it +is a runtime check on the resolved adapter, at both deploy and run, and it must +say what is wrong — a silent degradation that surfaces later as a connection +error is worse than no gate at all. + +One trap: `ee_oss::LICENSE_KEY_VALID` is initialized to `true` in the OSS +variant, so reading it alone passes on a CE build. The check is +`cfg!(feature = "enterprise") && LICENSE_KEY_VALID`, which rejects both a CE +build and an enterprise build whose key did not verify. + +## Decision 11: `dbt://`, keyed on the relation and not on the dbt node + +`dbt:////`, one `AssetKind`, where `` is the +workspace warehouse's NAME, so two scripts running against the same warehouse +agree on identity. + +The SCHEME names the producer, because dbt is the only thing that creates one of +these: no other language derives warehouse relations, `// materialize` takes +DuckLake targets only, and a dbt run does not dispatch. Calling the kind +something generic promised a parity with native Snowflake and BigQuery scripts +that does not exist. + +The PATH is the physical relation, and that is the load-bearing half. dbt-core +has no cross-project `ref()`: two projects meet when one materializes a mart and +the next declares it a `source`. Their dbt identities differ there — +`model.a_pkg.orders` against `source.b_pkg.analytics.orders` — while the relation +does not, so keying on `unique_id` would make every project an island and turn +the handoff into two unconnected nodes. `unique_id` also embeds the package name +from `dbt_project.yml`, which two unrelated projects may both call `analytics`, +collapsing two different tables onto one node. The relation cannot collide that +way. It is also what a DuckDB, Python, TS or Ansible script can name in a +`// on dbt://…` annotation to join the lineage — those four are the languages +with a body-asset parser; the native SQL ones cannot declare assets at all. + +A dbt run does **not** trigger those readers. See "no cascade from dbt" below. + +An ephemeral model (an inlined CTE, never written), an exposure, or a source that +is not separately modelled has no physical relation and therefore no place in +this namespace. If those ever prove worth rendering they need a key of their own +— `unique_id` suits them, precisely because nothing else can refer to them. + +Two traps, both of which quietly defeat the point if handled wrong. + +**Identifier canonicalization.** `manifest.json` gives `relation_name` +pre-quoted (`"windmill"."Analytics"."Orders"`), an annotation is written by hand, +and the warehouses disagree on case: Snowflake folds unquoted identifiers up, +Postgres folds them down, DuckDB compares case-insensitively. Two spellings of +one table produce two nodes, no edge, and nothing looks broken in isolation. So +one rule is applied in exactly one place — `parse_asset_syntax`, the single +point where an asset URI becomes a graph key: strip the quote characters +(`"`, backtick, `[`/`]`) from the schema and name, then ASCII-lowercase them, +matching the case-insensitive identifier comparison the DuckDB paths already +use. The warehouse-name prefix is spelled as the workspace configures it and +stays case-sensitive. + +**Warehouse identity is the workspace warehouse's name**, exactly as +`ducklake://main.orders` keys on the workspace lake's name — never the host, +account or database. A descriptor cannot name a resource at all (Decision 24), so +there is exactly one spelling per warehouse and the ambiguity a per-project +resource would create does not arise. The warehouse names the default database +too, so it stays out of the key; a model that *overrides* +its database (Snowflake `database`, BigQuery `project`) is genuinely elsewhere and +qualifies its schema segment as `.`, so two same-named relations +in different databases cannot collapse onto one node. A project that brings its own +`profiles.yml` reports its target's database from that file, read with the same +keys the renderer writes, so it spells a relation exactly as a workspace-warehouse +project does and the two meet on one node. Only where the target leaves its +database implicit does every relation qualify, because assuming they share one +database is exactly what would collapse them. + +Three call sites derive this key: the manifest ingest that creates the node, and +the live-progress and end-of-run paths that record status against it. They share +one function, because a site that derives it differently records progress against +a path no node has — the run still succeeds and the graph simply never moves. The same +warehouse is reachable under several hostnames, and credential material has no +business in an asset key. Accepted limitation, worth knowing before it is +filed as a bug: **two workspace warehouses pointing at the same physical +warehouse do not unify**, so assets under one will not share edges with assets +under the other. Point both projects at one warehouse to link them. + +## Decision 24: the warehouse is a workspace setting, named, and the only one + +A descriptor names a warehouse by NAME (`profile.warehouse`, `main` when it names +none) and cannot name a resource. Admins configure the warehouses under Settings +→ dbt, where each entry points at a resource, exactly as `large_file_storage` +points at the object-storage resource and a DuckLake names its catalog. + +Three things follow, and they are the reason for the rule rather than +consequences to work around. + +**A dbt project carries no connection.** The same project runs locally against a +developer's own `~/.dbt/profiles.yml` and on Windmill against the workspace +warehouse, with no Windmill-specific file in between and nothing to strip before +committing it to a repository. This is what makes Decision 23 possible at all: if +a project had to name its own resource, the descriptor could never be optional. + +**Asset identity has exactly one spelling.** Keying on a name is only sound +because a name is all there is. Had both `profile.resource` and +`profile.warehouse` existed, one physical warehouse would be reachable under two +spellings and two projects on it would silently fail to share nodes — the exact +failure Decision 11 exists to prevent. + +**dbt is unpermissioned, and the blast radius is bounded by construction +instead.** The warehouse resource is read with NO permission check on the runner, +exactly as `s3://` reaches the workspace bucket without the caller being granted +the storage resource: configuring a warehouse is what makes it available, and +anyone who may run a dbt script may build with it and read its models. What is +reachable stays bounded because only an admin writes the setting and a descriptor +cannot name a resource, only one of the names an admin configured. + +Per-relation rules were considered and rejected. `s3://` can enforce a path glob +because Windmill mediates every object operation through its proxy; dbt has no +such chokepoint — Windmill renders `profiles.yml` and dbt opens its own +connection. A rule could only be a pre-run check against the manifest, and a +`pre-hook`, a macro or `dbt run-operation` issues arbitrary SQL on the same +connection, so it would stop the ordinary case while implying a guarantee it +cannot keep. + +A project that brings its own `profiles.yml` still connects with it, and then +names a warehouse only to say where its assets belong. The name must still match +a configured warehouse — a typo is not identity, it strands the project's models +on a node nothing else reaches — but it grants nothing, since nothing here is +granted. It gets no identity by default, because defaulting to `main` would key a +self-hosted profile's tables onto a workspace warehouse it never connected to. + +That label is worth having only because such a project spells its relations the +same way: Windmill reads the target's database out of the project's own file +(Decision 11), so a mart it builds and a workspace-warehouse project's `source` +on the same relation land on ONE node. Without that the label would name a +namespace and still share nothing, which is the failure it exists to prevent. + +An agent worker cannot read the database, so it resolves the name through a +job-scoped API route. That route returns the resolved connection, which is why it +requires a job token: a running job already holds those credentials in its +rendered `profiles.yml`, and a browsable route would hand them to anyone. The +same worker posts its per-model outcomes to a second job-scoped route, since the +live reporter tails a log straight into the database and cannot run there. An +agent's run page therefore fills in when the run ends rather than during it. +Both routes are posted with the JOB's token: an agent's own credential +authenticates only against the agent surface. + +## Decision 23: the descriptor is optional, and lives inside the project + +` @@ -712,6 +721,10 @@ {/if} +{#if dbtRun} + +{/if} + {#if result_stream && result == undefined}
diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index d9692f4812..fabf9047c0 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -171,11 +171,15 @@ // For preserve_on_behalf_of feature let preserveOnBehalfOf = writable(false) let savedOnBehalfOfEmail = writable(savedFlow?.on_behalf_of_email) + let savedOnBehalfOfPermissionedAs = writable( + savedFlow?.on_behalf_of + ) // Keep savedOnBehalfOfEmail in sync when savedFlow is loaded asynchronously $effect(() => { if (savedFlow?.on_behalf_of_email !== undefined) { savedOnBehalfOfEmail.set(savedFlow.on_behalf_of_email) + savedOnBehalfOfPermissionedAs.set(savedFlow.on_behalf_of) } }) @@ -461,6 +465,7 @@ dedicated_worker: flow.dedicated_worker, visible_to_runner_only: flow.visible_to_runner_only, on_behalf_of_email: flow.on_behalf_of_email, + on_behalf_of: flow.on_behalf_of, preserve_on_behalf_of: $preserveOnBehalfOf || undefined, deployment_message: deploymentMsg || undefined, labels: (flow as any).labels @@ -509,6 +514,7 @@ ws_error_handler_muted: flow.ws_error_handler_muted, visible_to_runner_only: flow.visible_to_runner_only, on_behalf_of_email: flow.on_behalf_of_email, + on_behalf_of: flow.on_behalf_of, preserve_on_behalf_of: $preserveOnBehalfOf || undefined, deployment_message: deploymentMsg || undefined, labels: (flow as any).labels @@ -754,6 +760,7 @@ outputPickerOpenFns, preserveOnBehalfOf, savedOnBehalfOfEmail, + savedOnBehalfOfPermissionedAs, opWorkspace: () => opWorkspace }) diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index 73c9a08c4a..bb51851003 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -26,7 +26,10 @@ interface Props { code?: string - language: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined + // `sql` is the dialect-agnostic option: a dbt model's SQL is compiled by + // whichever adapter the project targets, so naming one dialect would be a + // guess. Every dialect below highlights through the same grammar anyway. + language: Script['language'] | 'bunnative' | 'frontend' | 'json' | 'sql' | undefined highlightLanguage?: LanguageType | undefined lines?: boolean className?: string @@ -56,7 +59,9 @@ ? 'opacity-0 group-hover:opacity-100 focus-within:opacity-100 transition-opacity duration-150' : '' - function getLang(lang: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined) { + function getLang( + lang: Script['language'] | 'bunnative' | 'frontend' | 'json' | 'sql' | undefined + ) { switch (lang) { case 'python3': return python @@ -76,6 +81,8 @@ return javascript case 'graphql': return graphql + case 'sql': + return sql case 'mysql': return sql case 'postgresql': diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index bd30cd6909..fcdb51fe32 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -43,6 +43,8 @@ import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' import { notifyContractWarnings } from './assets/AssetGraph/schemaContracts' import ScriptEditor from './ScriptEditor.svelte' + import DbtEditor from './dbt/DbtEditor.svelte' + import { findModulePathClash } from './scriptModulePath' import { Alert, Button, Drawer, SecondsInput, Tab, TabContent, Tabs } from './common' import LanguageIcon from './common/languageIcons/LanguageIcon.svelte' import type { SupportedLanguage, Schema } from '$lib/common' @@ -69,7 +71,7 @@ import { useLocalStorageValue } from '$lib/svelte5Utils.svelte' import { parsePipelineAnnotations } from './assets/AssetGraph/parsePipelineAnnotations' import DropdownV2 from './DropdownV2.svelte' - import { type Item } from '$lib/utils' + import { getLocalSetting, storeLocalSetting, type Item } from '$lib/utils' import { sendUserToast } from '$lib/toast' import { isCloudHosted } from '$lib/cloud' import Awareness from './Awareness.svelte' @@ -88,6 +90,8 @@ import { getContext, onMount, setContext, tick, untrack } from 'svelte' import EditorHeader from './EditorHeader.svelte' import ScriptSettingsBadges from './ScriptSettingsBadges.svelte' + import Badge from './common/badge/Badge.svelte' + import Modal from './common/modal/Modal.svelte' import AutosaveIndicator from './AutosaveIndicator.svelte' import LabelsInput from './LabelsInput.svelte' @@ -222,8 +226,10 @@ let isDeployer = $derived($userStore?.groups?.includes(WM_DEPLOYERS_GROUP) ?? false) let canPreserve = $derived(!!$userStore?.is_admin || !!$userStore?.is_super_admin || isDeployer) let originalOnBehalfOfEmail = $derived(savedScript?.on_behalf_of_email) + let originalOnBehalfOfPermissionedAs = $derived(savedScript?.on_behalf_of) let onBehalfOfChoice: OnBehalfOfChoice = $state(undefined) let customOnBehalfOfEmail: string = $state('') + let myPermissionedAs = $derived($userStore?.username ? `u/${$userStore.username}` : undefined) let metadataOpen = $state( !untrack(() => neverShowMeta) && @@ -235,7 +241,33 @@ ) let editor: Editor | undefined = $state(undefined) - let scriptEditor: ScriptEditor | undefined = $state(undefined) + // A dbt script gets its own editor: the artifact is a project bundle — a file + // tree, a descriptor, run arguments and a model graph — not a body of code. + // Both answer to the same handful of calls below, so everything around them + // is unchanged. + let scriptEditor: ScriptEditor | DbtEditor | undefined = $state(undefined) + let isDbt = $derived(script.language === 'dbt') + // dbt hides Generated UI, but the initialiser picks a tab before the language is + // known, so `ui` can be selected. Fall back only to an ENABLED tab — the other + // TabContents are not gated on their disable flags — and otherwise stay on `ui`, + // whose content IS gated for dbt, so nothing renders rather than something hidden. + $effect(() => { + if (isDbt && selectedTab === 'ui') { + const first = ( + [ + ['metadata', customUi?.settingsPanel?.disableMetadata], + ['runtime', customUi?.settingsPanel?.disableRuntime], + ['triggers', customUi?.settingsPanel?.disableTriggers] + ] as const + ).find(([, disabled]) => disabled !== true)?.[0] + if (first) selectedTab = first + } + }) + // The version whose stored graph the dbt editor draws until a refresh replaces + // it. A script never deployed has none, and `NewScript` carries no hash. + let deployedScriptHash = $derived( + savedScript && 'hash' in savedScript ? (savedScript.hash as string) : undefined + ) let captureTable: CaptureTable | undefined = $state(undefined) let wacExportDrawer: WacExportDrawer | undefined = $state(undefined) @@ -454,6 +486,8 @@ language: 'bun' } } + } else if (script.language === 'dbt') { + seedDbtProject() } const restarter = scheduleRestartSync(userDraftPath, { waitForContent: true }) initContent(script.language, script.kind, template).finally(() => restarter.markContentReady()) @@ -673,6 +707,7 @@ has_preprocessor: script.has_preprocessor, deployment_message: deploymentMsg || undefined, on_behalf_of_email: script.on_behalf_of_email, + on_behalf_of: script.on_behalf_of, preserve_on_behalf_of: preserveOnBehalfOf || undefined, assets: script.assets, modules: script.modules, @@ -971,7 +1006,48 @@ function handleDeployTrigger(_trigger: Trigger) {} + // A dbt script's modules ARE its dbt project, and the runtime refuses one + // without a `dbt_project.yml`. Seeded from BOTH entry points — the empty-script + // bootstrap and the language picker — because reaching dbt by switching an + // existing draft otherwise produces a script that cannot deploy or run. + // Existing modules are left alone: switching away and back must not discard a + // project the user has already grown. + function seedDbtProject() { + // Keyed on the project file rather than on "has any modules at all": a + // draft that grew modules under another language carries none of what dbt + // needs, and the worker refuses a bundle with no `dbt_project.yml` — so + // that draft reached dbt in a state it could neither deploy nor run. + // Matched on the canonical path: a bundle pushed with `./dbt_project.yml` + // already has the project file, and seeding a second spelling of it is the + // two-keys-one-file collision the editor's add-file checks refuse. + if (findModulePathClash(script.modules, 'dbt_project.yml')) return + script.modules = { + 'dbt_project.yml': { + content: + 'name: my_dbt_project\nversion: "1.0"\nprofile: my_dbt_project\nmodels:\n my_dbt_project:\n +materialized: view\n', + language: 'dbt' + }, + 'models/example.sql': { + content: 'select 1 as id\n', + language: 'dbt' + }, + // Last, so anything already written wins: the previous language's + // helper files are inert to dbt and are the user's to remove. + ...(script.modules ?? {}) + } + } + + /// Shown once, the first time dbt is chosen: the moment of choosing is when + /// "this is alpha and its details will move" is worth knowing, rather than + /// after a project has been built on it. + const DBT_ALPHA_SEEN = 'dbt_alpha_ack' + let dbtAlphaOpen = $state(false) + function onScriptLanguageTrigger(lang: 'docker' | 'bunnative' | ScriptLang) { + if (lang === 'dbt' && getLocalSetting(DBT_ALPHA_SEEN) !== 'true') { + dbtAlphaOpen = true + storeLocalSetting(DBT_ALPHA_SEEN, 'true') + } if (lang == 'docker') { template = 'docker' } else if (lang == 'bunnative') { @@ -980,9 +1056,19 @@ template = 'script' } let language = langToLanguage(lang) + if (language === 'dbt') { + // A project bundle only ever runs as an action, and the selector that + // would set this back is hidden for dbt — so a script arriving here as a + // trigger or approval would keep a kind it cannot fill and drop out of + // the action pickers, with nothing on screen to repair it. + script.kind = 'script' + } // initContent(language, script.kind, template) script.language = language + if (language === 'dbt') { + seedDbtProject() + } } function onSummaryChange(value: string) { @@ -1093,7 +1179,10 @@ label="Runtime" /> {/if} - {#if customUi?.settingsPanel?.disableGeneratedUi !== true} + + {#if customUi?.settingsPanel?.disableGeneratedUi !== true && !isDbt} {label} + {#if lang === 'dbt'} + alpha + {/if} {#snippet text()} {label} is only available with an enterprise license @@ -1350,7 +1442,10 @@ CI Test Python
- {#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true} + + {#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true && !isDbt}
{#snippet header()} { if (script.on_behalf_of_email) { script.on_behalf_of_email = undefined + script.on_behalf_of = undefined preserveOnBehalfOf = false onBehalfOfChoice = undefined } else { script.on_behalf_of_email = $userStore?.email + script.on_behalf_of = myPermissionedAs } }} options={{ @@ -1800,14 +1897,20 @@ onBehalfOfChoice = choice if (choice === 'me') { script.on_behalf_of_email = $userStore?.email + script.on_behalf_of = myPermissionedAs customOnBehalfOfEmail = '' preserveOnBehalfOf = false } else if (choice === 'target') { + // Keep the saved pair. A script that has no recorded principal yet + // sends the email alone and the backend derives one from it. script.on_behalf_of_email = originalOnBehalfOfEmail + script.on_behalf_of = + originalOnBehalfOfPermissionedAs customOnBehalfOfEmail = '' preserveOnBehalfOf = true } else if (choice === 'custom' && details) { script.on_behalf_of_email = details.email + script.on_behalf_of = details.permissionedAs customOnBehalfOfEmail = details.email preserveOnBehalfOf = true } @@ -1892,12 +1995,14 @@ {/if}
- - - + {#if !isDbt} + + + + {/if} {/if} - { - saveDraft() - }} - on:saveDraft={() => { - saveDraft() - }} - on:openTriggers={openTriggers} - on:applyArgs={applyArgs} - on:addPreprocessor={addPreprocessor} - bind:editor - bind:this={scriptEditor} - bind:schema={script.schema} - path={script.path} - stablePathForCaptures={initialPath || fakeInitialPath} - bind:code={script.content} - lang={script.language} - timeout={script.timeout} - kind={script.kind} - autoKind={script.auto_kind} - {template} - tag={script.tag} - lastSavedCode={savedScript?.content} - lastDeployedCode={savedScript?.content} - bind:args - bind:hasPreprocessor - bind:captureTable - bind:assets={script.assets} - bind:modules={script.modules} - enablePreprocessorSnippet - {testPanelCollapsed} - /> + {#if isDbt} + saveDraft()} + on:saveDraft={() => saveDraft()} + bind:editor + bind:this={scriptEditor} + bind:schema={script.schema} + path={script.path} + bind:code={script.content} + timeout={script.timeout} + tag={script.tag} + deployedHash={deployedScriptHash} + bind:args + bind:modules={script.modules} + /> + {:else} + { + saveDraft() + }} + on:saveDraft={() => { + saveDraft() + }} + on:openTriggers={openTriggers} + on:applyArgs={applyArgs} + on:addPreprocessor={addPreprocessor} + bind:editor + bind:this={scriptEditor} + bind:schema={script.schema} + path={script.path} + stablePathForCaptures={initialPath || fakeInitialPath} + bind:code={script.content} + lang={script.language} + timeout={script.timeout} + kind={script.kind} + autoKind={script.auto_kind} + {template} + tag={script.tag} + lastSavedCode={savedScript?.content} + lastDeployedCode={savedScript?.content} + bind:args + bind:hasPreprocessor + bind:captureTable + bind:assets={script.assets} + bind:modules={script.modules} + enablePreprocessorSnippet + {testPanelCollapsed} + /> + {/if} {:else} Script Builder not available to operators {/if} + +
+

+ A dbt script is a whole dbt project: the files are the script's module bundle, the content is + a wm_dbt.yaml descriptor, and the models become + dbt:// assets in the graph. +

+

+ Expect rough edges, and expect details to move: the descriptor's fields, what a run returns, + and how the graph is stored are all still settling. +

+

+ Deploying and running work today, and an existing project needs no changes — + cp -r it into the script's folder and push. Nothing here + is load-bearing for other languages. +

+

+ Docs + · report anything surprising, it is the most useful thing at this stage. +

+
+ + diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 9a65fc9d2b..dd06a58cde 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -19,6 +19,7 @@ import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow' import WacDiagram from '$lib/components/graph/WacDiagram.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' + import { canonicalModulePath, findModulePathClash } from './scriptModulePath' import SchemaForm from './SchemaForm.svelte' import PowerShellCommonParams from './PowerShellCommonParams.svelte' import LogPanel from './scriptEditor/LogPanel.svelte' @@ -354,6 +355,10 @@ editor?.setCode(editorCode) } + // Whether the open file is tested as a runnable of its own. A `__mod` helper + // is. + let onModuleArgs = $derived(activeModuleTab !== null) + let effectiveLang = $derived( activeModuleTab && modules?.[activeModuleTab] ? (modules[activeModuleTab].language as Preview['language']) @@ -476,25 +481,33 @@ function validateModulePath(path: string): string { if (!path.trim()) return '' - const moduleLang = inferModuleLang(path) + const canonical = canonicalModulePath(path) + if ('error' in canonical) return canonical.error + const moduleLang = inferModuleLang(canonical.path) if (!moduleLang) { const exts = allowedModuleExtensions.join(', ') return `File must end with a supported extension: ${exts}` } - const matchedExt = allowedModuleExtensions.find((ext) => path.endsWith(ext)) + const matchedExt = allowedModuleExtensions.find((ext) => canonical.path.endsWith(ext)) if (!matchedExt) { const exts = allowedModuleExtensions.join(', ') return `File must end with a supported extension for this language: ${exts}` } - if (modules?.[path.trim()]) { - return `Module ${path.trim()} already exists` + const clash = findModulePathClash(modules, canonical.path) + if (clash) { + return `Module ${clash} already exists` } return '' } function addModule() { - const modulePath = modulePathInput.trim() - if (!modulePath) return + if (!modulePathInput.trim()) return + const canonical = canonicalModulePath(modulePathInput) + if ('error' in canonical) { + modulePathError = canonical.error + return + } + const modulePath = canonical.path const error = validateModulePath(modulePath) if (error) { modulePathError = error @@ -525,27 +538,41 @@ function validateRenameModulePath(newPath: string, oldPath: string): string { if (!newPath.trim()) return '' - const moduleLang = inferModuleLang(newPath) + const canonical = canonicalModulePath(newPath) + if ('error' in canonical) return canonical.error + const moduleLang = inferModuleLang(canonical.path) if (!moduleLang) { const exts = allowedModuleExtensions.join(', ') return `File must end with a supported extension: ${exts}` } - const matchedExt = allowedModuleExtensions.find((ext) => newPath.endsWith(ext)) + const matchedExt = allowedModuleExtensions.find((ext) => canonical.path.endsWith(ext)) if (!matchedExt) { const exts = allowedModuleExtensions.join(', ') return `File must end with a supported extension for this language: ${exts}` } - if (newPath.trim() !== oldPath && modules?.[newPath.trim()]) { - return `Module ${newPath.trim()} already exists` + const clash = findModulePathClash(modules, canonical.path, oldPath) + if (clash) { + return `Module ${clash} already exists` } return '' } + /// A spelling of the name the module already has. Nothing to do, so the button + /// that would submit it stays disabled rather than being a dead click. + function renameIsNoop(input: string, oldPath: string): boolean { + const canonical = canonicalModulePath(input) + return 'path' in canonical && canonical.path === oldPath + } + function renameModule(oldPath: string) { - const newPath = renameModuleInput.trim() - if (!newPath || newPath === oldPath) { + if (!renameModuleInput.trim()) return + const canonical = canonicalModulePath(renameModuleInput) + if ('error' in canonical) { + renameModuleError = canonical.error return } + const newPath = canonical.path + if (newPath === oldPath) return const error = validateRenameModulePath(newPath, oldPath) if (error) { renameModuleError = error @@ -838,15 +865,15 @@ // Flush module edits back to modules map before running preview flushModuleContent() - const testCode = activeModuleTab !== null ? editorCode : code - const testLang = activeModuleTab !== null ? effectiveLang : lang - const rawTestArgs = - activeModuleTab !== null - ? testPanelArgs - : selectedTab === 'preprocessor' || kind === 'preprocessor' - ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...(args ?? {}) } - : (args ?? {}) - const testSchema = activeModuleTab !== null ? testPanelSchema : schema + const onModule = onModuleArgs + const testCode = onModule ? editorCode : code + const testLang = onModule ? effectiveLang : lang + const rawTestArgs = onModule + ? testPanelArgs + : selectedTab === 'preprocessor' || kind === 'preprocessor' + ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...(args ?? {}) } + : (args ?? {}) + const testSchema = onModule ? testPanelSchema : schema const testArgs = await processSecretArgs(rawTestArgs, testSchema, opWs) if (showPsCommonParams) { for (const [k, v] of Object.entries(psCommonParams)) { @@ -891,7 +918,8 @@ } }, undefined, - activeModuleTab !== null ? undefined : modules, + // A `__mod` helper is tested alone, so its siblings are left out. + onModule ? undefined : modules, undefined, timeout ) @@ -2241,7 +2269,7 @@ { if (e.detail) { - if (activeModuleTab !== null) { + if (onModuleArgs) { testPanelArgs = e.detail } else { args = e.detail @@ -2259,7 +2287,7 @@ bind:clientHeight={schemaHeight} > {#key argsRender} - {#if activeModuleTab !== null} + {#if onModuleArgs} + Test {/snippet} @@ -2371,7 +2401,7 @@ previewIsLoading={debugMode ? $debugState.running && !$debugState.stopped : testIsLoading} {editor} {diffEditor} - args={activeModuleTab !== null ? testPanelArgs : args} + args={onModuleArgs ? testPanelArgs : args} {showCaptures} customUi={customUi?.previewPanel} showCustomResultPanel={showDebugPanel} @@ -2497,7 +2527,7 @@ close() }} disabled={!renameModuleInput.trim() || - renameModuleInput.trim() === oldPath || + renameIsNoop(renameModuleInput, oldPath) || !!renameModuleError}>Rename @@ -2595,7 +2625,7 @@ {/if} -
+
{#if assets?.length} diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index f1c4daa962..9292e694b9 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -59,6 +59,11 @@ // import { createConfiguredEditor } from 'vscode/monaco' // import type { IStandaloneCodeEditor } from 'vscode/vscode/vs/editor/standalone/browser/standaloneCodeEditor' + /** Trailing debounce window (ms) on Monaco's onDidChangeModelContent. */ + const CHANGE_TIMEOUT = 200 + + let changeTimeoutId: number | undefined = undefined + let divEl: HTMLDivElement | null = null let editor = $state(null) let model: meditor.ITextModel @@ -98,7 +103,8 @@ readOnly = false, minHeight = 1000, renderLineHighlight = 'none', - suggestion + suggestion, + leadingChangeSync = false }: { lang: string code?: string @@ -130,6 +136,11 @@ minHeight?: number renderLineHighlight?: 'all' | 'line' | 'gutter' | 'none' suggestion?: string + /** Materialize `code` on the first change of a burst instead of only after + * the trailing debounce. Set it when a control's enabled state derives from + * `code`; leave it off where each extra sync costs work downstream (an app + * code input feeding an autoRefresh runnable re-runs a job per sync). */ + leadingChangeSync?: boolean } = $props() let yPadding = MONACO_Y_PADDING @@ -156,11 +167,21 @@ code = ncode } editor?.setValue(ncode) + // setValue emits a change event of its own; drop the burst it opens so an edit + // made right after an authoritative overwrite still counts as a leading change. + cancelPendingChanges() if (formatCode) { format() } } + function cancelPendingChanges(): void { + if (changeTimeoutId !== undefined) { + clearTimeout(changeTimeoutId) + changeTimeoutId = undefined + } + } + export function formatCode(): void { format() } @@ -408,12 +429,21 @@ pasteListenerCleanup = () => pasteTarget?.removeEventListener('keydown', onPasteKeydown, true) } - let timeoutModel: number | undefined = undefined - editor.onDidChangeModelContent((event) => { - timeoutModel && clearTimeout(timeoutModel) - timeoutModel = setTimeout(() => { + editor.onDidChangeModelContent(() => { + // A paste is a single change, so under a trailing-only sync `code` stays + // stale for CHANGE_TIMEOUT after it: a consumer gating a control on `code` + // (FlowYamlEditor disables "Apply changes" until it differs from a snapshot) + // then swallows a click made in that window. Schedule before firing so a + // re-entrant change from a consumer does not count as leading too. + const leading = leadingChangeSync && changeTimeoutId === undefined + cancelPendingChanges() + changeTimeoutId = setTimeout(() => { + changeTimeoutId = undefined updateCode() - }, 200) + }, CHANGE_TIMEOUT) + if (leading) { + updateCode() + } }) editor.onDidChangeCursorPosition((event) => { if (key) editorPositionMap[key] = event.position @@ -621,6 +651,7 @@ onDestroy(() => { try { valueAfterDispose = getCode() + cancelPendingChanges() pasteListenerCleanup?.() vimDisposable?.dispose() model && model.dispose() diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index 3763f7aeed..b84048455b 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -2,6 +2,7 @@ import { CornerDownLeft, Loader2 } from 'lucide-svelte' import Button from './common/button/Button.svelte' import { runScriptAndPollResult } from './jobs/utils' + import { writingJobOptions } from './jobs/writingJob' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { untrack } from 'svelte' @@ -123,7 +124,8 @@ args: dbArg } }, - { withJobData: true } + // The user types arbitrary SQL here, so treat every run as a write. + { withJobData: true, ...writingJobOptions } )) as any if (statements.length > 1) { result = result[result.length - 1] diff --git a/frontend/src/lib/components/WorkerRepl.svelte b/frontend/src/lib/components/WorkerRepl.svelte index a4a3e77ce9..180da6bb98 100644 --- a/frontend/src/lib/components/WorkerRepl.svelte +++ b/frontend/src/lib/components/WorkerRepl.svelte @@ -105,7 +105,12 @@ } }) - let result: any = await pollJobResult(jobId, $workspaceStore!) + // The shell tag is the worker's name prefix, which its shell loop pulls + // directly instead of advertising it in `worker_ping`, so the missing-worker + // check would read it as unserved. + let result: any = await pollJobResult(jobId, $workspaceStore!, { + failIfNoWorkerForTag: false + }) if (isOnlyCdCommand) { working_directory = (result as string).replace(/(\r\n|\n|\r)/g, '') @@ -365,9 +370,10 @@ > Full path - Commands run in the default directory. Run a standalone ‘cd’ to change it. Chained or invalid ‘cd’ commands won’t apply. + Commands run in the default directory. Run a standalone ‘cd’ to change it. Chained or + invalid ‘cd’ commands won’t apply.
diff --git a/frontend/src/lib/components/apps/components/display/dbtable/AppDbExplorer.svelte b/frontend/src/lib/components/apps/components/display/dbtable/AppDbExplorer.svelte index 8e068274b8..2696e0912c 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/AppDbExplorer.svelte +++ b/frontend/src/lib/components/apps/components/display/dbtable/AppDbExplorer.svelte @@ -399,7 +399,13 @@ resolvedConfig.type.configuration[selected].table ) - if (!tableMetadata) return + if (!tableMetadata) { + //@ts-ignore + gridItem.data.configuration.columnDefs.loading = false + gridItem.data = gridItem.data + $app = $app + return + } let old: TableMetadata = (columnDefs?.value as TableMetadata) ?? [] if (!Array.isArray(old)) { diff --git a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts index 1d0fa63342..b9af2ddda5 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts @@ -1,5 +1,3 @@ -import { JobService } from '$lib/gen' - import { runScriptAndPollResult } from '$lib/components/jobs/utils' import type { DbInput } from '$lib/components/dbTypes' import { @@ -43,50 +41,33 @@ export async function loadTableMetaData( // back to `DATABASE()`), so we don't read the resource value client-side for it. const content = makeMetadataMarker('LOAD_TABLE_METADATA', { table }, ducklake) - const job = await JobService.runScriptPreview({ - workspace, - requestBody: { language, content, args: dbArg } - }) + try { + const rows = (await runScriptAndPollResult({ + workspace, + requestBody: { language, content, args: dbArg } + })) as Record[] + const result = rows.map(lowercaseKeys) - const maxRetries = 8 - let attempts = 0 - while (attempts < maxRetries) { - try { - await new Promise((resolve) => setTimeout(resolve, 1000 * (attempts || 0.6))) - - const testResult = (await JobService.getCompletedJob({ - workspace, - id: job - })) as any - - if (testResult.success) { - attempts = maxRetries - - const result = testResult.result.map(lowercaseKeys) - - // For Snowflake, fetch primary keys separately - if ( - input.type === 'database' && - (input.resourceType === 'snowflake' || (input.resourceType as any) === 'snowflake_oauth') - ) { - const map: Record = { [table]: result } - await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace, table) - return map[table] - } - - return result - } else { - attempts++ - } - } catch (error) { - attempts++ + // For Snowflake, fetch primary keys separately + if ( + input.type === 'database' && + (input.resourceType === 'snowflake' || (input.resourceType as any) === 'snowflake_oauth') + ) { + const map: Record = { [table]: result } + await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace, table) + return map[table] } - } - console.error('Failed to load table metadata after maximum retries.') - return undefined + return result + } catch (e) { + console.error('Failed to load table metadata', e) + sendUserToast('Error loading table metadata: ' + ((e as Error)?.message || e), true) + return undefined + } } +/** Throws on failure without reporting it: every caller renders the error in its + * own pane, so toasting here would double-report it. */ export async function loadAllTablesMetaData( workspace: string | undefined, input: DbInput @@ -120,7 +101,7 @@ export async function loadAllTablesMetaData( return map } catch (e) { - sendUserToast('Error loading tables metadata: ' + e, 'error') + console.error('Failed to load tables metadata', e) throw e } } @@ -171,7 +152,7 @@ async function fetchSnowflakePrimaryKeys( const payload: Record = {} if (tableKey) payload.table = tableKey const content = makeMetadataMarker('SNOWFLAKE_PRIMARY_KEYS', payload, undefined) - return (await JobService.runScriptPreviewAndWaitResult({ + return (await runScriptAndPollResult({ workspace, requestBody: { language: 'snowflake', @@ -206,7 +187,7 @@ export async function getDbSchemas( let result: unknown try { - result = await JobService.runScriptPreviewAndWaitResult({ + result = await runScriptAndPollResult({ workspace, requestBody: { language: sqlScript.lang as Preview['language'], diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte index edaefe947a..214ab3e31d 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte @@ -7,7 +7,7 @@ import { inferArgs } from '$lib/infer' import { initialCode } from '$lib/script_helpers' import { emptySchema } from '$lib/utils' - import { defaultScriptLanguages, getScriptByPath, processLangs } from '$lib/scripts' + import { defaultScriptLanguages, getScriptByPath, processInlineLangs } from '$lib/scripts' import { Building, GitFork, Globe2 } from 'lucide-svelte' import { createEventDispatcher } from 'svelte' @@ -88,7 +88,7 @@ } let langs = $derived( - processLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) + processInlineLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) .map((l) => [defaultScriptLanguages[l], l]) .filter( (x) => diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 3e5e9d8cef..27f6ce6be7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -18,10 +18,15 @@ import PanToNode from './PanToNode.svelte' import InitialFitView from './InitialFitView.svelte' import { layoutAssetGraph } from './assetGraphLayout' - import { computeMutedReadKeys } from './resolveGraph' + import { computeMutedReadKeys, dbtAssociations } from './resolveGraph' import { buildDownstreamMap } from './graphTraversal' import { buildLineageDownstreamMap } from './boundedCascade' - import type { AssetGraphResponse, AssetGraphSelection, NativeTriggerKind } from './types' + import type { + AssetGraphResponse, + AssetGraphSelection, + AssetRunState, + NativeTriggerKind + } from './types' import type { RunnableRunState } from './activeRunnables.svelte' import type { AssetKind } from '$lib/gen' import { NODE } from '$lib/components/graph/util' @@ -179,6 +184,11 @@ * When a node's nonce changes, it flashes a fading green background — its * producer just recomputed it. Driven by the replay player frame-by-frame. */ recomputedAssetIds?: ReadonlyMap + /** What a run is currently doing to each relation, keyed `asset::` + * like every other per-asset map here. + * Distinct from `recomputedAssetIds`, which is a one-shot pulse: this is + * the state a node holds until the run moves it. */ + assetRunStatus?: ReadonlyMap /** Let the wheel zoom the canvas (and swallow the page scroll while doing * so). Default true for the full-height editor/player. Set false when the * canvas is embedded inline inside a scrollable container, so a wheel @@ -215,6 +225,7 @@ viewportFitKey = '', highlightActiveRun = false, recomputedAssetIds, + assetRunStatus, scrollZoom = true }: Props = $props() @@ -244,6 +255,7 @@ | 'data-test' | 'macro' | 'test-dependency' + | 'dbt-ref' unsaved?: boolean // Muted read edge: a ducklake/s3 input read every run whose (default) // auto cascade trigger is suppressed by `// mute` / `// mute all`. @@ -287,6 +299,26 @@ // by node id across producers). const addedTestNodes = new Set() + // A dbt script owns every relation its project materializes. Drawing that + // as one edge per model buries the lineage that matters — `ref()` between + // models, and native consumers — under a fan-out that grows with the + // project, so the association is carried by the model's badge and its + // hover/click highlight instead. Only the DRAWING is dropped: the + // producer rows still drive cascade dispatch and "who produced this". + const dbtRunnableIds = new Set( + g.runnables.filter((r) => r.dbt).map((r) => `${r.usage_kind}:${r.path}`) + ) + // Association only — the canvas deliberately draws no edge for it. + const { ownerByAsset: dbtOwnerByAsset, writesByOwner: dbtWritesByOwner } = dbtAssociations( + g.runnables, + g.edges + ) + // Per-relation dbt description, to tell a project's own declared source + // from a relation another script materializes. + const dbtAssetProvenance = new Map( + g.assets.filter((a) => a.dbt).map((a) => [`asset:${a.kind}:${a.path}`, a.dbt!]) + ) + const hasAddNode = onAddPipelineScript != null if (hasAddNode) { nodes.push({ @@ -395,6 +427,7 @@ path: a.path, fork_materialization: a.fork_materialization, derived_from: a.derived_from, + dbt: a.dbt, onAddScript: onAddScriptForAsset, pathPrefix, defaultPathSuffix, @@ -404,7 +437,32 @@ producerFailed, // Bumped by the replay player when this asset's producer just // recomputed it — the node flashes green and fades. - recomputePulse: recomputedAssetIds?.get(assetId) + recomputePulse: recomputedAssetIds?.get(assetId), + // What the run in view is doing to this relation right now. + runStatus: assetRunStatus?.get(assetId)?.status, + runRowCount: assetRunStatus?.get(assetId)?.rowCount, + // The dbt project that materializes this relation, related by + // badge rather than by an edge — so only when that node is on + // this graph. The run page and the pipeline page both hide it, + // and passing handlers anyway makes the chip advertise a click + // that resolves to nothing. + ...(dbtOwnerByAsset.has(assetId) + ? { + onDbtHover: (on: boolean) => (dbtHoverId = on ? assetId : undefined), + onDbtSelect: () => { + // `runnable::` — the id shape `build` uses. + const owner = model.dbtOwnerByAsset.get(assetId) + const [kind, ...rest] = owner?.split(':') ?? [] + if (kind && rest.length) { + onselect?.({ + kind: 'runnable', + runnable_kind: kind as 'script' | 'flow', + path: rest.join(':') + }) + } + } + } + : {}) } }) } @@ -475,6 +533,8 @@ tag: r.tag, retry: r.retry, macros: r.macros, + dbt: r.dbt, + onDbtHover: (on: boolean) => (dbtHoverId = on ? rid : undefined), unsaved: r.unsaved ?? false, // Same dispatch the asset node uses, only routed when the // runnable is a script (the page handler short-circuits @@ -522,10 +582,28 @@ // (`// mute` / `// mute all` opted the default auto trigger out). Gated // on pipeline scripts inside the helper (non-pipeline reads never derive). const mutedReadKeys = computeMutedReadKeys(g.edges, g.triggers, g.runnables) + // A dbt script owns every relation of its project. Drawing that as one + // edge per model buries the lineage that matters (`ref()` between models, + // and native consumers) under a fan-out that grows with the project — so + // the association is carried by the node badge and its hover/click + // highlight instead. Only the DRAWING is dropped: the producer rows still + // drive cascade dispatch and "who produced this". for (const e of g.edges) { const runnableId = `${e.runnable_kind}:${e.runnable_path}` const assetId = `asset:${e.asset_kind}:${e.asset_path}` const access = e.access_type ?? 'r' + // A dbt project's own relations are related by badge, not by edges: its + // writes are the fan-out, and its declared sources already reach its + // models through the `ref()` edges, so both would be noise. + // + // A read of a relation ANOTHER script builds is different — that is how + // two selections of one project compose (decision 6), it carries the + // cascade, and no `ref()` edge survives the split to stand in for it. + // Kept, or the two halves render as disconnected islands. + if (dbtRunnableIds.has(runnableId)) { + const isOwnSource = dbtAssetProvenance.get(assetId)?.resource_type === 'source' + if (access === 'w' || access === 'rw' || isOwnSource) continue + } if (access === 'w' || access === 'rw') { // Data tests assert on the `// materialize` target, which is always // a ducklake asset (v1 enforces this), so only the ducklake @@ -621,6 +699,17 @@ }) } + // dbt `ref()` lineage: model → model inside one project. The dbt script + // writes every one of them, so without these the canvas shows a flat + // fan-out from the script and loses the project's actual shape. + const assetNodeIds = new Set(g.assets.map((a) => `asset:${a.kind}:${a.path}`)) + for (const de of g.dbt_edges ?? []) { + const from = `asset:dbt:${de.from_asset_path}` + const to = `asset:dbt:${de.to_asset_path}` + if (!assetNodeIds.has(from) || !assetNodeIds.has(to)) continue + edges.push({ id: `dbtref:${from}->${to}`, source: from, target: to, kind: 'dbt-ref' }) + } + // Non-asset triggers (schedule + native) are rendered as source nodes // above the pipeline script. Real (non-missing) nodes are // deduplicated per (kind, ref) tuple so a single schedule shared @@ -787,11 +876,24 @@ } } - return { nodes, edges } + return { nodes, edges, dbtOwnerByAsset, dbtWritesByOwner } } let model = $derived(build(graph)) + // dbt association, surfaced by emphasis instead of edges. Hovering a model's + // dbt badge lights up the project node that materializes it; hovering the + // project node lights up every model it owns. Clicking the badge selects the + // project node, so the association survives the pointer leaving. + let dbtHoverId = $state(undefined) + let dbtEmphasisIds = $derived.by(() => { + if (!dbtHoverId) return new Set() + const owned = model.dbtWritesByOwner.get(dbtHoverId) + if (owned) return new Set([dbtHoverId, ...owned]) + const owner = model.dbtOwnerByAsset.get(dbtHoverId) + return owner ? new Set([dbtHoverId, owner]) : new Set() + }) + let selectedId = $derived.by(() => { if (!selection) return undefined return selection.kind === 'asset' @@ -895,12 +997,13 @@ else if (boundPick.bounded.has(n.id)) boundClass = 'wm-bound-in' else if (!boundPick.eligible.has(n.id)) boundClass = 'wm-bound-dim' } + const dbtClass = dbtEmphasisIds.has(n.id) ? 'wm-dbt-linked' : undefined return { id: n.id, type: n.type, position: { x: p.x + xCenter + xShift, y: p.y + 40 }, data: n.data, - class: boundClass ?? runClass ?? assetClass, + class: boundClass ?? dbtClass ?? runClass ?? assetClass, selected: n.id === selectedId, // All nodes non-draggable: the layout is sugiyama-computed, // dragging would fight the reactive re-layout. Selection is @@ -1064,6 +1167,21 @@ label = 'test needs' labelStyle = 'fill: rgb(217 119 6); font-size: 10px; font-weight: 600;' break + case 'dbt-ref': + // model → model inside one dbt project. Orange, matching the + // dbt badges, and dashed because the edge is dbt's own lineage + // rather than a Windmill read/write the cascade acts on. + style = 'stroke: rgb(234 88 12); stroke-width: 1.25px;' + strokeDasharray = '4 3' + markerColor = 'rgb(234 88 12)' + label = 'ref' + labelStyle = 'fill: rgb(234 88 12); font-size: 10px; font-weight: 600;' + // Same rule the pipeline uses for a running script: the edges + // touching what is happening animate. Here the unit of work is + // the model, so the edges feeding the one dbt is building move, + // and the flow reads in DAG order as it advances. + animated = assetRunStatus?.get(e.target)?.status === 'running' + break default: style = '' } @@ -1181,9 +1299,32 @@ } // 'schedule' doesn't produce a selection. } + + /// Clicking empty canvas clears the selection. `onselect` already promises it + /// can receive `undefined`, it just never did — so a details pane opened from + /// a node had no way to close by clicking off it. + /// + /// Read off the click TARGET rather than SvelteFlow's `onpaneclick`, which + /// does not fire here: what counts as "empty" is everything that is not a + /// node, an edge or one of the canvas's own controls, and that is a question + /// about the DOM the click landed on. In the CAPTURE phase, because the flow + /// stops the click before it bubbles back out to this wrapper. + function handleBackgroundClick(event: MouseEvent) { + if (boundPick || !onselect) return + const t = event.target as HTMLElement | null + if ( + t?.closest( + '.svelte-flow__node, .svelte-flow__edge, .svelte-flow__controls, .svelte-flow__minimap, .svelte-flow__panel' + ) + ) { + return + } + onselect(undefined) + } -
+ +
{ + const file = selectionDbt?.original_file_path + if (!file) return undefined + // A relation may have several script producers, and nothing here says which + // of them is the dbt project this model came from. Prefixing the wrong one + // names a `__dbt` folder that does not exist, so an ambiguous relation shows + // the path inside the project alone. + const scripts = selectionProducers.filter((p) => p.kind === 'script') + return scripts.length === 1 ? `${scripts[0].path}__dbt/${file}` : file + }) + // Bound from ScriptEditor — populated by inferAssets on every code // change. Forwarded to the page so the canvas can re-derive write // edges as the user edits the body (e.g. renaming a CREATE TABLE @@ -1216,6 +1236,25 @@
{/key} + {:else if selectionDbt?.raw_code} + +
+
+ + {dbtBundlePath ?? selectionDbt.unique_id} + read-only · edit locally +
+
+ +
+
{:else}
No inline preview yet for {selection.asset_kind}. Use the producer/consumer arrows diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte index 999559a0c7..815db36b18 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -14,13 +14,17 @@ Loader2, Plus, ShieldCheck, - ShieldAlert + ShieldAlert, + CheckCircle2, + XCircle } from 'lucide-svelte' import type { ScriptLang } from '$lib/gen' import { enterpriseLicense, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/utils' import { PIPELINE_LANGUAGES } from './pipelineLanguages' import type { PipelineOutputKind } from './pipelineTemplates' + import type { DbtAssetProvenance } from './types' + import DbtIcon from '$lib/components/icons/DbtIcon.svelte' // Shape used for both the data prop and the run callback. Drafts carry // `content` / `language` so the page-level run handler can dispatch to @@ -44,6 +48,15 @@ // "current view of " marker so it reads as a derived node, not an // unrelated table. derived_from?: string + // dbt provenance when this warehouse table is a dbt node: which model + // it is, how dbt materializes it, its tags and its generic tests. + dbt?: DbtAssetProvenance + /** Hovering the dbt chip emphasizes the project node that + * materializes this model — the association the graph deliberately + * does not draw as an edge. */ + onDbtHover?: (on: boolean) => void + /** Clicking it selects that project node. */ + onDbtSelect?: () => void onAddScript?: ( asset: { kind: AssetKind; path: string }, language: ScriptLang, @@ -78,6 +91,11 @@ // producer just recomputed it. A change triggers a one-shot green // fade so a freshly-written table stands out as the run progresses. recomputePulse?: number + // What the run being viewed is doing to this relation. dbt records it + // per model as it walks the DAG, so the graph moves with the run + // instead of only settling once the job ends. + runStatus?: 'running' | 'materialized' | 'failed' + runRowCount?: number | null } // SvelteFlow injects this on the node component when the user clicks // the node. Combined with our own `hovered` state to drive the @@ -133,6 +151,40 @@ let showAdd = $derived(data.onAddScript != undefined) + // dbt badge. `materialized` is dbt's own word rather than the Windmill + // strategy because `view` and `ephemeral` have no strategy, and showing the + // dbt word keeps the node legible to someone reading their own project. + let dbtLabel = $derived(data.dbt?.materialized ?? data.dbt?.resource_type) + let dbtTitle = $derived.by(() => { + const d = data.dbt + if (!d) return '' + const lines = [`dbt ${d.resource_type}: ${d.unique_id}`] + if (d.materialized) { + const strategy = d.materialize_strategy ? ` -> ${d.materialize_strategy}` : '' + lines.push(`materialized: ${d.materialized}${strategy}`) + } + if (d.tags?.length) lines.push(`tags: ${d.tags.join(', ')}`) + for (const t of d.data_tests ?? []) { + const col = t.column ? ` on ${t.column}` : '' + lines.push(`test ${t.kind}${col}${t.severity ? ` [${t.severity}]` : ''}`) + } + const cols = Object.entries(d.columns ?? {}) + if (cols.length) { + lines.push( + `columns: ${cols.map(([c, desc]) => (desc ? `${c} (${desc})` : c)).join(', ')}` + ) + } + if (d.freshness) { + const f = d.freshness as Record + const window = (k: string) => + f[k]?.count != null ? `${k.replace('_after', '')} after ${f[k].count}${f[k].period?.[0] ?? ''}` : '' + const windows = ['warn_after', 'error_after'].map(window).filter(Boolean) + if (windows.length) lines.push(`freshness: ${windows.join(', ')}`) + } + if (d.description) lines.push(d.description) + return lines.join('\n') + }) + // Data-test outcome badge. Only guarded assets show it. The write's fate on a // failing test differs by edition — surface which one applies so a shared // parent/fork table name can't hide a silently-published bad version. @@ -200,6 +252,39 @@ class={`shrink-0 ml-2 mr-2 ${selected ? 'text-accent' : 'text-blue-600 dark:text-blue-400'}`} size="14px" /> + {#if data.runStatus} + + + {#if data.runStatus === 'running'} + + {:else if data.runStatus === 'failed'} + + {:else} + + {/if} + + + {#if data.runStatus === 'materialized' && data.runRowCount != undefined} + + {Intl.NumberFormat().format(data.runRowCount)} + + {/if} + {/if} {formatShortAssetPath(asset)} @@ -225,6 +310,34 @@ fork {/if} + + {#if data.dbt} + + {/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte index 751e149266..1b86fa6853 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte @@ -102,6 +102,11 @@ : undefined ) + // Kept apart from "no columns for this table": a failed metadata read says + // nothing about whether the table exists, so reporting it as missing would + // point the user at the wrong problem. + let colDefsError = $state(undefined) + // Metadata for every table in the datatable. We query the lot rather // than just the one we care about because `loadAllTablesMetaData` is // what `dbTableOpsWithPreviewScripts` and the rest of the DB manager @@ -111,14 +116,12 @@ let colDefs = resource( () => [input, refreshKey], async ([_input]) => { + colDefsError = undefined if (!_input || !$workspaceStore) return undefined try { return await loadAllTablesMetaData($workspaceStore, _input) - } catch { - // Connection/permission errors look identical to "table - // missing" from the user's POV inside the preview pane; - // the full error is surfaced via the existing sendUserToast - // path inside loadAllTablesMetaData. + } catch (e) { + colDefsError = (e as Error)?.message || String(e) return undefined } } @@ -187,6 +190,12 @@
+ {:else if colDefsError} +
+ + Could not read the datatable + {colDefsError} +
{:else if !tableColDefs} diff --git a/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte index 403a100aa2..f21ca486af 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte @@ -53,15 +53,19 @@ scoped && partition ? `_wm_partition = '${partition.replaceAll("'", "''")}'` : undefined ) + // Distinct from "no columns for this table": a failed metadata read says + // nothing about whether the table exists. + let colDefsError = $state(undefined) + let colDefs = resource( () => [input, refreshKey] as const, async ([_input]) => { + colDefsError = undefined if (!_input || !$workspaceStore) return undefined try { return await loadAllTablesMetaData($workspaceStore, _input) - } catch { - // A load failure reads the same as "table missing" from the preview's - // POV; the underlying error is surfaced by loadAllTablesMetaData. + } catch (e) { + colDefsError = (e as Error)?.message || String(e) return undefined } } @@ -119,6 +123,11 @@
+ {:else if colDefsError} +
+ + {colDefsError} +
{:else if !tableColDefs}
diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 7db35c2353..ec28b7c784 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -16,8 +16,7 @@ AssetGraphResponse, AssetGraphSelection, NativeTriggerKind, - PipelineMode - } from './types' + PipelineMode, DbtAssetProvenance } from './types' import type { AssetKind, Script, ScriptLang } from '$lib/gen' import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte' import type { PipelineOutputKind } from './pipelineTemplates' @@ -76,6 +75,7 @@ localScriptsVersion, selectionProducers = [], selectionColumnGraph, + selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, schemaContractContext = undefined, @@ -181,6 +181,8 @@ selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> /** Transitive column-lineage trace for a selected ducklake asset (route page). */ selectionColumnGraph?: ColumnLineageGraph + /** dbt provenance of the selected relation — carries its SQL. */ + selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean /** Fork workspaces: data-environment state of the selected ducklake asset (route page). */ selectionForkMaterialization?: 'fork' | 'deferred' @@ -512,6 +514,7 @@ selection={activeDraft ? undefined : editor.selection} selectionProducers={activeDraft ? [] : selectionProducers} {selectionColumnGraph} + {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} {schemaContractContext} diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index d8a977539b..c260bc2e5b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -18,6 +18,7 @@ XCircle, Zap } from 'lucide-svelte' + import DbtIcon from '$lib/components/icons/DbtIcon.svelte' import { twMerge } from 'tailwind-merge' import { preventDefault, stopPropagation } from 'svelte/legacy' import type { GraphUsageKind } from './types' @@ -45,6 +46,13 @@ // Macros this script provides (deployed/drafted `// macros` library). // Non-empty renders the ƒ chip marking the node as a macro library. macros?: { name: string; params: string; is_table: boolean }[] + // Set on a dbt script: the number of models the project materializes. + // One runnable node stands for the whole project, so the count is what + // tells it apart from a single-output script. + dbt?: { model_count: number } + /** Hovering the project badge emphasizes every model it + * materializes — the fan-out the graph deliberately omits. */ + onDbtHover?: (on: boolean) => void // Last-run status + run count observed this session (from the // folder queue poll). Undefined until the first observed run. runState?: RunnableRunState @@ -188,7 +196,19 @@ -
(hover = true)} onmouseleave={() => (hover = false)}> + +
{ + hover = true + data.onDbtHover?.(true) + }} + onmouseleave={() => { + hover = false + data.onDbtHover?.(false) + }} +> +
+ + {logOrDate.log.username} + {#if logOrDate.log.parameters && 'end_user' in logOrDate.log.parameters} - ({logOrDate.log.parameters.end_user}) + + ({logOrDate.log.parameters.end_user}) + {/if}
+ {:else} +
+ + + + + {#snippet trigger()} + + {#if argsOverridden} + + {/if} + {/snippet} + {#snippet content()} + {#if schema} + + {:else} +

This descriptor takes no arguments.

+ {/if} + {/snippet} + +
+ {/if} +
+
+ + + + +
+ + {#snippet addFile()} + + {#snippet trigger()} +
+ +
+ {/snippet} + {#snippet content({ close })} +
+ + (newFileError = dbtPathError(newFile, modules ?? undefined))} + onkeydown={(e) => { + if (e.key === 'Enter') createFile() + if (e.key === 'Escape') close() + }} + /> + {#if newFileError} +

{newFileError}

+ {/if} +

+ Anywhere in the project, e.g. macros/cents.sql. + {DBT_MODULE_EXTENSIONS.join(', ')} +

+
+ + +
+
+ {/snippet} + + {/snippet} + +
+ {#key fileLang} + { + if (openFile === null) { + code = editorCode + lastSyncedCode = code + inferSchema(editorCode) + } else { + flushOpenFile() + } + }} + on:saveDraft + cmdEnterAction={async () => { + if (openFile === null) await inferSchema(editorCode) + runTest() + }} + formatAction={async () => { + if (openFile === null) await inferSchema(editorCode) + dispatch('format') + }} + class="flex flex-1 h-full !overflow-visible" + scriptLang={fileLang} + automaticLayout={true} + fixedOverflowWidgets={true} + {args} + customTag={tag} + /> + {/key} +
+
+ + + + + + { + graphSelection = sel + selectedDbt = dbt + selectedBuffer = buffer + }} + /> + + + + {#if selectedAsset && selectedDbt} + (graphSelection = undefined)} + /> + {/if} +
+ +
+ + + + + diff --git a/frontend/src/lib/components/dbt/DbtModelDetails.svelte b/frontend/src/lib/components/dbt/DbtModelDetails.svelte new file mode 100644 index 0000000000..2d25dbdc00 --- /dev/null +++ b/frontend/src/lib/components/dbt/DbtModelDetails.svelte @@ -0,0 +1,323 @@ + + +
+
+ {assetPath.split('/').pop()} + {#if dbt.materialized} + {dbt.materialized} + {/if} + {#if dbt.resource_type !== 'model'} + {dbt.resource_type} + {/if} + {#each dbt.tags ?? [] as t (t)} + {t} + {/each} +
+ {#if previewable} + {#if showRows && preview && !('error' in preview)} + + {:else} + + {/if} + {/if} + {#if fileInBundle && dbt.original_file_path && onOpenFile} + + {/if} + +
+
+ + {#if stalePlaceholders} +
+ The run arguments have changed since this graph was parsed, so these rows need not + describe the models on screen — arguments reach schemas, aliases and which models exist + at all. Refresh the models to draw and preview them under the current ones. +
+ {:else if staleVars} +
+ The run form's vars have changed since this graph was parsed. Rows are previewed under + the vars it was parsed with, so they still describe the models on screen — refresh the + models to draw and preview them under the current ones. +
+ {/if} + +
+
+
+ relation + {assetPath} +
+ {#if dbt.original_file_path} +
+ file + {dbt.original_file_path} +
+ {/if} + {#if dbt.description} +
+ description + {dbt.description} +
+ {/if} +
+ + {#if columns.length > 0 || (dbt.data_tests?.length ?? 0) > 0} +
+ {#if columns.length > 0} +
+
columns declared
+
+ {#each columns as [name, desc] (name)} +
+ {name} + {desc} +
+ {/each} +
+ +
+ Declared metadata — dbt reports no column-level lineage. +
+
+ {/if} + {#if (dbt.data_tests?.length ?? 0) > 0} +
+
tests
+
+ {#each dbt.data_tests ?? [] as t, i (i)} + + {t.kind}{t.column ? ` · ${t.column}` : ''} + + {/each} +
+
+ {/if} +
+ {/if} + + {#if showRows && preview} + {#if 'error' in preview} +
{preview.error}
+ {#if dbt.raw_code} + + {/if} + {:else if 'pending' in preview} +
+ + Running `dbt show` — this is a job, so it waits on a worker and the engine. +
+ {:else} + {@const cols = Object.keys(preview.rows[0] ?? {})} + {#if cols.length === 0} +
The model returned no rows.
+ {:else} + + + + {#each cols as c (c)} + + {/each} + + + + {#each preview.rows as row, i (i)} + + {#each cols as c (c)} + + {/each} + + {/each} + +
{c}
{cellText(row[c])}
+
+ {preview.rows.length} rows in {(preview.tookMs / 1000).toFixed(1)}s{preview.node + ? ` · ${preview.node}` + : ''} +
+ {/if} + {/if} + {:else if dbt.raw_code} + + {:else} +
+ {dbt.resource_type === 'source' + ? 'A source is declared rather than built, so it has no transform of its own.' + : 'No SQL stored for this node.'} +
+ {/if} +
+
diff --git a/frontend/src/lib/components/dbt/DbtModelGraph.svelte b/frontend/src/lib/components/dbt/DbtModelGraph.svelte new file mode 100644 index 0000000000..e6d948d926 --- /dev/null +++ b/frontend/src/lib/components/dbt/DbtModelGraph.svelte @@ -0,0 +1,504 @@ + + +
+
+ + {provenance} + + {#if modelCount > 0} + · {modelCount} models + {/if} +
+ +
+
+ + {#if refreshPending} +
+ Still parsing. A cold worker provisions the dbt engine before it starts; a project + pinned to a worker tag nothing serves waits here indefinitely. + Open the parse job +
+ {/if} + + {#if refreshError} +
+
The parse failed
+ 
{refreshError.message}
+ {#if refreshError.job} + Open the parse job + {/if} +
+ {/if} + + {#if loading} +
+ Loading the model graph +
+ {:else if failed} +
Could not load the model graph.
+ {:else if !graph} +
+ {#if refreshJob || deployedHash != undefined} + + This project has no models in the asset graph. A project that brings its own + profiles.yml without naming a + profile.warehouse has no warehouse identity to key + dbt:// assets on, so its relations cannot be drawn. + + {:else} + + Nothing has parsed this project yet. Refresh models runs + dbt parse over the files as they are here and draws what dbt + reports — deploying does the same. + + + It needs a warehouse it can reach: the profile is rendered before dbt runs, so a project + whose profile.warehouse is not configured under Settings → dbt + fails the parse the way it would fail a run. + + {/if} +
+ {:else} +
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/dbt/DbtProjectPanel.svelte b/frontend/src/lib/components/dbt/DbtProjectPanel.svelte new file mode 100644 index 0000000000..89b275aab3 --- /dev/null +++ b/frontend/src/lib/components/dbt/DbtProjectPanel.svelte @@ -0,0 +1,162 @@ + + +{#snippet branch(nodes: Node[], depth: number)} + {#each nodes as node (node.name + node.path + depth)} + {@const key = `${depth}:${node.name}:${node.path}`} + {#if node.children.length > 0} + + {#if !collapsed[key]} + {@render branch(node.children, depth + 1)} + {/if} + {:else} +
+ + + {#if onDelete && node.path !== projectFileKey} + + {/if} +
+ {/if} + {/each} +{/snippet} + +
+
+ {scriptPath}__dbt/ +
+ {fileCount + 1} + {@render addFile?.()} +
+
+
+ + + {@render branch(tree, 0)} +
+ {#if fileCount === 0} +
+ No project yet. Copy one in and push it: + 
cp -r my-dbt-project/. {scriptPath}__dbt/
+wmill sync push
+
+ {/if} +
diff --git a/frontend/src/lib/components/dbt/DbtRunGraph.svelte b/frontend/src/lib/components/dbt/DbtRunGraph.svelte new file mode 100644 index 0000000000..cffb663b44 --- /dev/null +++ b/frontend/src/lib/components/dbt/DbtRunGraph.svelte @@ -0,0 +1,774 @@ + + +{#snippet sqlPane()} + {#if selectedIsForeign} +
+ Another dbt project in this workspace also materializes this relation, and the graph keeps one + project's model per relation — so the SQL shown here would not be this run's. Open that + project's own run to see it. +
+ {:else if selectedDbt?.raw_code} +
+
+ {selectedDbt.original_file_path ?? selectedDbt.unique_id} + {#if selectedDbt.materialized} + {selectedDbt.materialized} + {/if} + + {#if selectedDbt.resource_type === 'model'} + {#if showRows && preview && !('error' in preview)} + + {:else} + + {/if} + {/if} + {#if selectedRelation} + + {/if} + + + read-only · edit in the script + +
+
+ {#if showRows && preview} + {#if 'error' in preview} +
{preview.error}
+ + {:else if 'pending' in preview} +
+ + Running `dbt show` — this is a job, so it waits on a worker and the engine. +
+ {:else} + {@const cols = Object.keys(preview.rows[0] ?? {})} + {#if cols.length === 0} +
The model returned no rows.
+ {:else} + + + + {#each cols as c (c)} + + {/each} + + + + {#each preview.rows as row, i (i)} + + {#each cols as c (c)} + + {/each} + + {/each} + +
{c}
{cellText(row[c])}
+
+ {preview.rows.length} rows in {(preview.tookMs / 1000).toFixed(1)}s{preview.node + ? ` · ${preview.node}` + : ''} +
+ {/if} + {/if} + {:else} + + {/if} +
+
+ {/if} +{/snippet} + +{#if resumable} +
+ {(run?.totals?.error ?? 0) > 0 ? `${run?.totals?.error} failed` : ''}{(run?.totals?.error ?? + 0) > 0 && (run?.totals?.skipped ?? 0) > 0 + ? ', ' + : ''}{(run?.totals?.skipped ?? 0) > 0 ? `${run?.totals?.skipped} skipped` : ''}. Rebuild only + those with dbt retry, instead of the whole project. + + + +
+{/if} + +{#if loading} +
+ Loading the model graph +
+{:else if failed} +
Could not load the model graph.
+{:else if !graph} +
+ {#if ranTestsOnly} + This run selected tests alone, so it built no models. A dbt test is an assertion rather than a + relation, so it has no node here — the models it asserts against belong to the runs that build + them. Its results are in the table below. + {:else} + + No models stored for this version of the project. + {#if scriptHash} + A version's graph is written by its deploy, so this is what a deploy still in + flight looks like — it fills in when that job lands. + {/if} + A project that brings its own profiles.yml without naming + a profile.warehouse also has no warehouse identity to key + models on, and stores none. + {/if} +
+{:else} +
+ {#if relationDrift > 0} +
+ {relationDrift} + {relationDrift === 1 ? 'model has' : 'models have'} been renamed or moved since this run — + {relationDrift === 1 ? 'its node shows' : 'their nodes show'} today's relation, not the one this + run wrote. +
+ {/if} + {#if goneSinceRun > 0} +
+ {goneSinceRun} + {goneSinceRun === 1 ? 'model' : 'models'} this run built {goneSinceRun === 1 ? 'is' : 'are'} + no longer in the project — renamed or removed since, so + {goneSinceRun === 1 ? 'it is' : 'they are'} not drawn. +
+ {/if} +
+ (selection = s)} + showMinimap={false} + scrollZoom={false} + /> +
+ {@render sqlPane()} +
+{/if} diff --git a/frontend/src/lib/components/dbt/DbtRunResult.svelte b/frontend/src/lib/components/dbt/DbtRunResult.svelte new file mode 100644 index 0000000000..34d7886b30 --- /dev/null +++ b/frontend/src/lib/components/dbt/DbtRunResult.svelte @@ -0,0 +1,146 @@ + + +
+
+ {#each [{ k: 'success', label: 'passed', cls: 'text-green-600 dark:text-green-400' }, { k: 'warn', label: 'warned', cls: 'text-yellow-600 dark:text-yellow-400' }, { k: 'error', label: 'failed', cls: 'text-red-600 dark:text-red-400' }, { k: 'skipped', label: 'skipped', cls: 'text-secondary' }] as t (t.k)} + {@const n = (totals as Record)[t.k] ?? 0} + {#if n > 0} + {n} {t.label} + {/if} + {/each} + of {totals.total ?? nodes.length} nodes + + {run.command ?? 'build'} · {run.engine ?? ''} + {run.engine_version ?? ''} + +
+ + {#if nodes.length > 0} +
+ + + + + + + + + + + + {#each nodes as node (node.unique_id)} + {@const s = split(node.unique_id)} + {@const r = rank(node.status, node.outcome)} + + + + + + + + {/each} + +
NodeKindRelationRowsTime
+
+ + {#if r === 0} + + {:else if r === 1} + + {:else if r === 2} + + {:else} + + {/if} + + {s.name} + {#if node.message && r < 2} + {node.message} + {/if} +
+ {#if node.message && r < 2} +
+ {node.message} +
+ {/if} + 		+ + {#if s.kind === 'test'} + + {/if} + {s.kind} + + + {fmtRelation(node.relation_name) ?? ''} + + {node.rows_affected ?? ''} + + {fmtTime(node.execution_time)} +
+
+ {#if hasTests} +
+ A test's severity decides the outcome: dbt's own warn surfaces + without failing the job. +
+ {/if} + {/if} +
diff --git a/frontend/src/lib/components/dbt/parseDbtRun.test.ts b/frontend/src/lib/components/dbt/parseDbtRun.test.ts new file mode 100644 index 0000000000..47d6073ef7 --- /dev/null +++ b/frontend/src/lib/components/dbt/parseDbtRun.test.ts @@ -0,0 +1,193 @@ +import { describe, it, expect } from 'vitest' +import { + parseDbtRun, + relationOutcome, + splitRelation, + statusRank, + splitUniqueId, + nodeSelector +} from './parseDbtRun' + +const run = { + engine: 'dbt-core-1x', + engine_version: '1.12.0', + command: 'build', + totals: { total: 1, success: 1, error: 0, warn: 0, skipped: 0 }, + nodes: [{ unique_id: 'model.p.customers', status: 'success' }] +} + +describe('parseDbtRun', () => { + it('takes a successful run as-is', () => { + expect(parseDbtRun(run)?.engine).toBe('dbt-core-1x') + }) + + // The worker puts the same JSON in the error message after the exit-status + // line, and this is the case worth rendering: the failing node is what the + // user came for. + it('recovers the run from a failed job’s error message', () => { + const failed = { + error: { + name: 'ExecutionErr', + message: `execution error:\nNon-zero exit status for dbt build: 1\n\n${JSON.stringify(run)}` + } + } + expect(parseDbtRun(failed)?.totals?.total).toBe(1) + }) + + // The failures worth reading are the ones whose message carries braces of its + // own — a Jinja template, the compiled SQL, an adapter's own JSON — and the + // payload is appended after all of it. + it('finds the run past braces in the error text', () => { + const failed = { + error: { + name: 'ExecutionErr', + message: + 'execution error:\nCompilation Error in model x\n {{ ref("missing") }} depends on {"a": 1}\n' + + `Non-zero exit status for dbt build: 1\n\n${JSON.stringify(run)}` + } + } + expect(parseDbtRun(failed)?.totals?.total).toBe(1) + }) + + // `{nodes, totals}` alone is a shape an ordinary script can return, and it + // would then be rendered as somebody's dbt run. + it('does not claim an ordinary result that happens to have nodes and totals', () => { + expect(parseDbtRun({ nodes: [], totals: {} })).toBeUndefined() + expect(parseDbtRun({ engine: 'v8', nodes: [], totals: {} })).toBeUndefined() + }) + + it('accepts every engine the worker stamps', () => { + for (const engine of ['dbt-core-1x', 'dbt-core-2x', 'fusion']) { + expect(parseDbtRun({ ...run, engine })?.engine).toBe(engine) + } + }) + + // The payload carries one object per node, so a scan bounded by brace COUNT + // gives up on an ordinary project — a few hundred nodes, tests included — and + // silently loses the per-model table on exactly the runs it exists for. + it('finds the run in a payload with hundreds of nodes', () => { + const big = { + ...run, + totals: { total: 400, success: 399, error: 1, warn: 0, skipped: 0 }, + nodes: Array.from({ length: 400 }, (_, i) => ({ + unique_id: `model.p.m${i}`, + status: i === 0 ? 'error' : 'success' + })) + } + const failed = { + error: { + name: 'ExecutionErr', + message: + 'execution error:\nCompilation Error {{ ref("x") }} {"a": 1}\n\n' + + JSON.stringify(big, null, 2) + } + } + expect(parseDbtRun(failed)?.nodes?.length).toBe(400) + }) + + it('is undefined for anything unparseable', () => { + expect(parseDbtRun(undefined)).toBeUndefined() + expect(parseDbtRun('a string')).toBeUndefined() + expect(parseDbtRun({ error: { message: 'failed with {not json' } })).toBeUndefined() + }) +}) + +describe('statusRank', () => { + // dbt counts `partial success` in totals.error and a retry redoes it, so + // ranking it as a pass would contradict the job's own outcome. + it('ranks partial success with the failures', () => { + expect(statusRank('partial success')).toBe(statusRank('error')) + expect(statusRank('PARTIAL SUCCESS')).toBe(0) + }) + + // The worker publishes `unknown` for a status it does not recognise and counts + // it in `totals.error`; ranking it as a pass drew a green check on a node the + // same result called an error. + // The worker counts `no_op` in totals.skipped — dbt built nothing for that + // node — so a green check would claim a run that never happened. + it('ranks a no-op with the skips, not with the passes', () => { + expect(statusRank('no-op', 'no_op')).toBe(statusRank('skipped', 'skipped')) + expect(statusRank('success', 'no_op')).toBe(2) + }) + + it('ranks an unknown outcome with the failures, not with the passes', () => { + expect(statusRank('some-future-dbt-status', 'unknown')).toBe(0) + expect(statusRank('success', 'unknown')).toBe(0) + }) + + it('orders failed before warned before skipped before passed', () => { + expect( + ['success', 'skipped', 'warn', 'error'].sort((a, b) => statusRank(a) - statusRank(b)) + ).toEqual(['error', 'warn', 'skipped', 'success']) + }) +}) + +describe('splitUniqueId', () => { + it('splits kind from name and drops a generic test’s uniqueness hash', () => { + expect(splitUniqueId('model.jaffle.stg_orders')).toEqual({ + kind: 'model', + name: 'stg_orders' + }) + expect(splitUniqueId('test.jaffle.not_null_orders_id.4e687af8d0')).toEqual({ + kind: 'test', + name: 'not_null_orders_id' + }) + // A model whose name contains a dot keeps it: only tests carry the hash. + expect(splitUniqueId('model.jaffle.a.b').name).toBe('a.b') + }) +}) + +describe('relationOutcome', () => { + it('agrees with the worker classifier on every status it names', () => { + expect(relationOutcome('started')).toBe('running') + for (const s of ['success', 'pass', 'PASS', ' Success ']) { + expect(relationOutcome(s)).toBe('materialized') + } + // `partial success` built the relation and then failed its tests; the + // worker records it failed, so the colour must agree. + for (const s of ['error', 'fail', 'runtime error', 'partial success', 'PARTIAL SUCCESS']) { + expect(relationOutcome(s)).toBe('failed') + } + // Nothing was built, so nothing is coloured. + for (const s of ['warn', 'skipped', 'no-op', 'something new']) { + expect(relationOutcome(s)).toBeUndefined() + } + }) +}) + +describe('splitRelation', () => { + it('keeps a period that lives inside a quoted identifier', () => { + // The backend supports it, so rendering it as `v2.orders` names a + // relation that does not exist. + expect(splitRelation('"wh"."analytics.v2"."orders"')).toEqual(['wh', 'analytics.v2', 'orders']) + expect(splitRelation('"db"."schema"."name"')).toEqual(['db', 'schema', 'name']) + expect(splitRelation('db.schema.name')).toEqual(['db', 'schema', 'name']) + // BigQuery backticks and T-SQL brackets quote too. + expect(splitRelation('`proj`.`data.set`.`t`')).toEqual(['proj', 'data.set', 't']) + expect(splitRelation('[db].[my.schema].[t]')).toEqual(['db', 'my.schema', 't']) + }) + + // Every one of these dialects escapes its delimiter by doubling it. Dropping + // the pair renames the relation, and the manifest keeps the real spelling — + // so the run's status would be recorded against a key no graph node has. + it('keeps a delimiter the identifier escaped by doubling', () => { + expect(splitRelation('"wh"."schema"."a""b"')).toEqual(['wh', 'schema', 'a"b']) + expect(splitRelation('`proj`.`da``ta`.`t`')).toEqual(['proj', 'da`ta', 't']) + expect(splitRelation('[db].[my]]schema].[t]')).toEqual(['db', 'my]schema', 't']) + }) +}) + +describe('nodeSelector', () => { + // Verified against dbt-core 1.12, dbt-core 2.0.0-alpha.5 and fusion + // 2.0.0-preview.202: the intersection resolves to the one node whatever the + // project's `model-paths` is, while a path-derived FQN resolves to nothing + // as soon as that root is more than one segment deep. + it('intersects the name with its package, wherever the model sits', () => { + expect(nodeSelector('model.jaffle_shop.fct_orders')).toBe('fct_orders,package:jaffle_shop') + }) + + // Ambiguous across packages, but a selector dbt resolves rather than rejects. + it('falls back to the bare name without a package', () => { + expect(nodeSelector('fct_orders')).toBe('fct_orders') + }) +}) diff --git a/frontend/src/lib/components/dbt/parseDbtRun.ts b/frontend/src/lib/components/dbt/parseDbtRun.ts new file mode 100644 index 0000000000..aecd3dcf03 --- /dev/null +++ b/frontend/src/lib/components/dbt/parseDbtRun.ts @@ -0,0 +1,247 @@ +export type DbtNode = { + unique_id: string + status: string + /** Windmill's stable word for the same result, published beside dbt's own. + * Preferred wherever a decision is made: `status` is dbt's vocabulary and + * dbt may rename it. */ + outcome?: DbtOutcome + execution_time?: number + rows_affected?: number + relation_name?: string + message?: string +} + +export type DbtRun = { + engine?: string + engine_version?: string + command?: string + totals?: { total?: number; success?: number; error?: number; warn?: number; skipped?: number } + nodes?: DbtNode[] + /** The arguments the run actually used, as submitted. A `dbt retry` restores + * the failed run's arguments inside the worker, so the retry job's own args + * name only the run it resumed — this is the sole way to recover what it + * really ran with. */ + invocation_args?: Record +} + +/** The engines the worker stamps on a result. This is the discriminator: a + * `{nodes, totals}` shape alone is one an ordinary script can return, and it + * would then be rendered as somebody's dbt run. */ +const ENGINES = ['dbt-core-1x', 'dbt-core-2x', 'fusion'] + +function asDbtRun(v: unknown): DbtRun | undefined { + if (!v || typeof v !== 'object') return undefined + const o = v as Record + return ENGINES.includes(o.engine as string) && + Array.isArray(o.nodes) && + o.totals != undefined && + typeof o.totals === 'object' + ? (o as DbtRun) + : undefined +} + +/** + * The dbt invocation a job result describes, if it describes one. + * + * On success the result IS the run. On failure the worker puts the same JSON in + * the error message after the exit-status line, and that is the case worth + * rendering: the failing node is what the user came for. + */ +export function parseDbtRun(result: any): DbtRun | undefined { + const direct = asDbtRun(result) + if (direct) return direct + const msg = result?.error?.message + if (typeof msg !== 'string') return undefined + // The payload is appended pretty-printed, so its `{` is the only one at COLUMN + // ZERO — everything nested is indented, and dbt's own error text carries its + // braces mid-line. Counting braces instead needs a cap that a real project + // blows: forwards on an error full of them, backwards on one `{` per node. + for (const line of lineStarts(msg)) { + if (msg[line] !== '{') continue + try { + const run = asDbtRun(JSON.parse(msg.slice(line))) + if (run) return run + } catch { + // A `{` alone on a line inside the error text; the payload is later. + } + } + return undefined +} + +/** Index of the first character of each line, the payload's own `{` among them. */ +function* lineStarts(s: string): Generator { + let at = 0 + while (at !== -1 && at < s.length) { + yield at + const next = s.indexOf('\n', at) + at = next === -1 ? -1 : next + 1 + } +} + +/** Ordering rank of a node's status: 0 failed, 1 warned, 2 skipped, 3 passed. + * + * Both `unknown` and `no_op` rank where the RESULT counts them, not where the + * default would put them: the worker counts `unknown` in `totals.error` and + * `no_op` in `totals.skipped`, so falling through to 3 drew a green check on a + * node the same result called an error, and on one it never built. */ +export function statusRank(status: string, outcome?: DbtOutcome): number { + switch (outcome ?? classifyStatus(status)) { + case 'failed': + case 'unknown': + return 0 + case 'warned': + return 1 + case 'skipped': + case 'no_op': + return 2 + default: + return 3 + } +} + +/** The worker's stable vocabulary for a node result, published as `outcome`. */ +export type DbtOutcome = + | 'started' + | 'passed' + | 'failed' + | 'warned' + | 'skipped' + | 'no_op' + | 'unknown' + +/** + * dbt's node status, reduced to the outcomes the UI distinguishes. + * + * Only for results that predate `outcome`, or for the live event stream, which + * carries dbt's word alone. Anything holding a node from a job result should + * read `outcome` instead — that is the field the worker publishes precisely so + * this mapping is not the contract. + */ +function classifyStatus( + status: string +): 'started' | 'passed' | 'failed' | 'warned' | 'skipped' | 'other' { + // `partial success` is dbt's word for a node that built but whose tests + // failed. The worker counts it in `totals.error` and a retry redoes it, so + // showing it green would contradict the job's own outcome. + switch (status.trim().toLowerCase()) { + case 'started': + return 'started' + case 'success': + case 'pass': + return 'passed' + case 'error': + case 'fail': + case 'runtime error': + case 'partial success': + return 'failed' + case 'warn': + return 'warned' + case 'skipped': + return 'skipped' + default: + return 'other' + } +} + +/** + * The kind and name behind a dbt `unique_id`, which dbt builds as + * `..`. A generic test's name carries a trailing + * hash dbt adds for uniqueness; it is noise in a run summary. + */ +export function splitUniqueId(uniqueId: string): { kind: string; name: string } { + const parts = uniqueId.split('.') + const kind = parts[0] ?? '' + let name = parts.slice(2).join('.') + if (kind === 'test') name = name.replace(/\.[0-9a-f]{6,}$/, '') + return { kind, name: name || uniqueId } +} + +/** + * What a node's status says happened to the relation it builds, or `undefined` + * when it says nothing. + * + * Mirrors the worker's `classify_status`, and must keep mirroring it: the two + * decide the same thing about the same string, one for the record it writes and + * one for the colour drawn over it. `warn`, `skipped` and `no-op` leave the + * relation untouched, so they get no colour rather than a misleading one. + */ +export function relationOutcome( + status: string, + outcome?: DbtOutcome +): 'running' | 'materialized' | 'failed' | undefined { + switch (outcome ?? classifyStatus(status)) { + case 'started': + return 'running' + case 'passed': + return 'materialized' + case 'failed': + return 'failed' + // `warn`, `skipped` and `no-op` say nothing about the relation: nothing + // was written, so its state is whatever the last run left. + default: + return undefined + } +} + +/** + * dbt's `relation_name` split into its parts, honouring quoting. + * + * Mirrors the worker's `split_relation`: `"`, `` ` `` and `[` open a quoted + * identifier, and a `.` inside one is part of the name. Splitting on every `.` + * turns `"wh"."analytics.v2"."orders"` into a relation called `orders` in a + * schema called `v2` — a table that does not exist. + */ +export function splitRelation(relation: string): string[] { + const parts: string[] = [] + let current = '' + let quote: string | undefined + for (let i = 0; i < relation.length; i++) { + const c = relation[i] + if (quote !== undefined) { + const close = quote === '[' ? ']' : quote + if (c === close) { + // Doubled, which is how each of these dialects escapes its own + // delimiter: one literal character, not the end of the identifier. + if (relation[i + 1] === close) { + current += close + i++ + } else { + quote = undefined + } + } else current += c + } else if (c === '"' || c === '`' || c === '[') { + quote = c + } else if (c === '.') { + parts.push(current) + current = '' + } else { + current += c + } + } + parts.push(current) + return parts.map((p) => p.trim()) +} + +/** + * A selector naming exactly one node: `,package:`. + * + * The comma is dbt's intersection operator, so this reads "the node whose name + * is `` and whose package is ``" — one node, since dbt refuses + * two models of one name inside a package. A bare name would match the leaf of + * every package's FQN, and a package can ship a model whose name the project + * also uses. + * + * Not the FQN (`..`), which cannot be rebuilt from + * `original_file_path`: how many leading segments are the resource root is + * `model-paths`, and dropping exactly one turns `src/models/marts/orders.sql` + * into `pkg.models.marts.orders`, which dbt's matcher — equal lengths, from the + * front — resolves to nothing at all. + * + * Without a package, the bare name — ambiguous across packages, but a selector + * dbt resolves rather than one it rejects. + */ +export function nodeSelector(uniqueId: string): string { + const { name } = splitUniqueId(uniqueId) + const pkg = uniqueId.split('.')[1] + return pkg ? `${name},package:${pkg}` : name +} diff --git a/frontend/src/lib/components/dbt/previewRows.ts b/frontend/src/lib/components/dbt/previewRows.ts new file mode 100644 index 0000000000..9364196d4a --- /dev/null +++ b/frontend/src/lib/components/dbt/previewRows.ts @@ -0,0 +1,108 @@ +import { JobService, type ScriptModule } from '$lib/gen' + +/** An unsaved project, submitted as its own preview job. Held as it was sent, + * never re-read from the editor: a graph describes the project it was parsed + * from, and so must anything run against that graph's nodes. */ +export type DbtPreviewBuffer = { + content: string + modules: Record | undefined + tag?: string + timeout?: number + /** The arguments the project was parsed under, in run-form shape. Part of the + * snapshot because vars decide `enabled`, schemas and aliases: run a preview + * under later ones and it can address a relation this graph never had. */ + args: Record | undefined +} + +/** A model's rows, as `dbt show` returns them. */ +export type DbtPreview = + | { pending: true } + | { rows: Record[]; node?: string; tookMs: number } + | { error: string } + +/** + * Preview one model's rows by running its own project's `dbt show`. + * + * A job, not a query: the rows come from the warehouse through the project's + * profile, with its vars and its adapter, which is the only place that knows how + * to resolve `ref()` and where the relation actually lives. `show` is therefore + * not a run-form command — it is what a table's preview is made of, here and on + * the run page's graph. + * + * Which project runs is decided by the caller, and must match the graph the SQL + * on screen came from: a deployed version by hash, a buffer parse by shipping + * that same buffer. + * + * `stillWanted` is asked before each poll and before the result is used, so a + * preview outlives neither the page that asked for it nor a navigation. + */ +export async function previewDbtRows(opts: { + workspace: string + scriptPath: string + /** Pins the preview to a deployed version, for a graph showing that version. */ + scriptHash?: string | number + /** The project the graph was parsed from, when that was a buffer. Required + * then, because there may be no deployed version at all — and when there + * is, it can lack the model or build it from other SQL. */ + buffer?: DbtPreviewBuffer + /** One node: a model name, or `package.model` where two packages share one. */ + model: string + /** The run's own vars, so a descriptor with a required `{{ }}` var resolves. */ + vars?: Record + limit?: number + /** Extra top-level arguments — a run's `{{ placeholder }}` values. */ + args?: Record + stillWanted?: () => boolean +}): Promise { + const { workspace, scriptPath, scriptHash, buffer, model, vars, limit, args, stillWanted } = opts + const startedAt = Date.now() + const requestBody = { + ...(args ?? {}), + command: { label: 'show', vars: vars ?? {}, model, limit: limit ?? 25 } + } + try { + // The BUFFER when the caller has one, because then the graph and the SQL + // above these rows are the buffer's: a model added since the deploy exists + // in no other project, and one whose SQL changed builds different rows + // there. It ships whole — dbt resolves `ref()` project-wide. + // + // Otherwise by HASH whenever the caller pins one: the SQL on screen is that + // version's, and running the deployed one would show today's rows under + // it — or fail outright for a model since removed. + const id = buffer + ? await JobService.runScriptPreview({ + workspace, + timeout: buffer.timeout, + requestBody: { + path: scriptPath, + content: buffer.content, + language: 'dbt', + tag: buffer.tag, + modules: buffer.modules, + args: requestBody + } + }) + : scriptHash + ? await JobService.runScriptByHash({ + workspace, + hash: String(scriptHash), + requestBody + }) + : await JobService.runScriptByPath({ workspace, path: scriptPath, requestBody }) + // Polled rather than awaited: a preview is a job, and its engine may need + // provisioning on a cold worker. + for (let i = 0; i < 90; i++) { + await new Promise((r) => setTimeout(r, 1000)) + if (stillWanted && !stillWanted()) return undefined + const done = await JobService.getCompletedJobResultMaybe({ workspace, id }) + if (!done.completed) continue + const res = done.result as { node?: string; show?: Record[] } | undefined + return done.success && res?.show + ? { rows: res.show, node: res.node, tookMs: Date.now() - startedAt } + : { error: 'The preview job failed — open it from Runs for the detail.' } + } + return { error: 'The preview is still running; open it from Runs.' } + } catch (e) { + return { error: e instanceof Error ? e.message : String(e) } + } +} diff --git a/frontend/src/lib/components/dbt/projectFiles.test.ts b/frontend/src/lib/components/dbt/projectFiles.test.ts new file mode 100644 index 0000000000..d9ed58b41a --- /dev/null +++ b/frontend/src/lib/components/dbt/projectFiles.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest' +import type { ScriptModule } from '$lib/gen' +import { dbtModelSelector, dbtModulePath, dbtProjectFileKey } from './projectFiles' + +const mod = (content: string): ScriptModule => ({ content, language: 'dbt' as any }) + +const PROJECT = `name: jaffle +model-paths: ["models"] +` + +describe('dbtModulePath', () => { + // The worker resolves `.` and `//` away when it materialises the bundle, so a + // redundant spelling is a second key for one file on disk and walks straight + // past a check that compares the typed string. + it('canonicalises before the reserved-name and duplicate checks', () => { + expect(dbtModulePath('./models//x.sql', {})).toEqual({ path: 'models/x.sql' }) + expect(dbtModulePath('./wm_dbt.yaml', {})).toEqual({ + error: expect.stringContaining('is the descriptor') + }) + const bundle = { './models/x.sql': mod('select 1') } + expect(dbtModulePath('models/x.sql', bundle)).toEqual({ + error: expect.stringContaining('./models/x.sql already exists') + }) + }) + + // A path outside the bundle has no canonical form inside it, and the worker + // drops it — which would read as a file that was added and never written. + it('refuses a path that escapes the bundle', () => { + expect(dbtModulePath('../secrets.sql', {})).toEqual({ error: expect.stringContaining('..') }) + expect(dbtModulePath('/etc/x.sql', {})).toEqual({ error: expect.stringContaining('relative') }) + }) + + it('refuses an extension dbt does not read', () => { + expect(dbtModulePath('models/x.txt', {})).toEqual({ + error: expect.stringContaining('must end with') + }) + }) +}) + +// Every write path canonicalises, so a read that compares the constant exactly +// would neither find a project imported under a redundant spelling nor protect +// it from deletion. +it('resolves the key a bundle actually holds the project file under', () => { + expect(dbtProjectFileKey({ './dbt_project.yml': mod(PROJECT) })).toBe('./dbt_project.yml') + expect(dbtProjectFileKey({})).toBeUndefined() +}) + +describe('dbtModelSelector', () => { + // Package-qualified, because a bare leaf name also matches a dependency + // package's model of the same name. + it('selects a model under the project’s own model-paths', () => { + const bundle = { 'dbt_project.yml': mod(PROJECT), 'models/orders.sql': mod('select 1') } + expect(dbtModelSelector(bundle, 'models/orders.sql')).toBe('orders,package:jaffle') + }) + + // A project may put its models anywhere; a macro or singular test is `.sql` + // too and is not selectable by name, so those fall back to the whole project. + it('honours a custom model-paths and skips what is not a model', () => { + const bundle = { + 'dbt_project.yml': mod('name: jaffle\nmodel-paths: ["transform"]\n'), + 'transform/orders.sql': mod('select 1'), + 'macros/cents.sql': mod('{% macro cents() %}{% endmacro %}') + } + expect(dbtModelSelector(bundle, 'transform/orders.sql')).toBe('orders,package:jaffle') + expect(dbtModelSelector(bundle, 'macros/cents.sql')).toBeUndefined() + }) + + it('finds the project file under a redundant spelling', () => { + const bundle = { './dbt_project.yml': mod(PROJECT), 'models/orders.sql': mod('select 1') } + expect(dbtModelSelector(bundle, 'models/orders.sql')).toBe('orders,package:jaffle') + }) +}) diff --git a/frontend/src/lib/components/dbt/projectFiles.ts b/frontend/src/lib/components/dbt/projectFiles.ts new file mode 100644 index 0000000000..36902ac74a --- /dev/null +++ b/frontend/src/lib/components/dbt/projectFiles.ts @@ -0,0 +1,162 @@ +/** + * The rules for the files of a dbt project, which is what a dbt script's module + * bundle is. + * + * A `__mod` bundle is homogeneous — one language, one extension — and a dbt + * project is not: models are SQL or Python, schemas and the project file YAML, + * seeds CSV, doc blocks Markdown. Every module is stored with `language: dbt` + * because they are dbt's to read, so everything below is decided by the file + * name. + */ +import type { Preview, ScriptModule } from '$lib/gen' +import { canonicalModulePath, findModulePathClash } from '../scriptModulePath' +import YAML from 'yaml' + +/** The descriptor. It is the script's CONTENT, not a module: a module at the + * same path would be a second, independent value for one file, since the export + * writes the content there and the bundle would emit over it. */ +export const DBT_DESCRIPTOR = 'wm_dbt.yaml' + +/** What makes the bundle a project. The worker refuses a version without it, so + * it is the one file the tree will not let you delete. */ +export const DBT_PROJECT_FILE = 'dbt_project.yml' + +/** The key a bundle actually holds `dbt_project.yml` under. + * + * Not the constant: nothing on the push path rewrites existing keys, so a + * project imported with `./dbt_project.yml` in it holds that spelling, and an + * exact lookup would neither find the project nor protect it from deletion — + * while every write path canonicalises. Resolved the same way the duplicate + * check resolves a clash. */ +export function dbtProjectFileKey( + modules: Record | null | undefined +): string | undefined { + return findModulePathClash(modules, DBT_PROJECT_FILE) +} + +/** + * Extensions a dbt project's own files take. `.py` because dbt Python models are + * first-class on Snowflake, BigQuery and Databricks. + */ +export const DBT_MODULE_EXTENSIONS = ['.sql', '.py', '.yml', '.yaml', '.csv', '.md'] + +/** + * The editor language for one project file, by extension. `postgresql` and + * `ansible` are how Windmill spells "SQL" and "YAML" to the editor. + */ +export function dbtFileLang(path: string): Preview['language'] { + if (path.endsWith('.sql')) return 'postgresql' + if (path.endsWith('.yml') || path.endsWith('.yaml')) return 'ansible' + if (path.endsWith('.py')) return 'python3' + // `.md` (a doc block) and `.csv` (a seed) have no grammar of their own here; + // `bash` leaves prose alone, where the default would colour it as TypeScript. + return 'bash' +} + +/** Whether this file may be added to the bundle at all. */ +export function dbtModuleLang(filePath: string): ScriptModule['language'] | undefined { + return DBT_MODULE_EXTENSIONS.some((e) => filePath.endsWith(e)) + ? ('dbt' as ScriptModule['language']) + : undefined +} + +/** The canonical key a typed path becomes, or the reason it cannot be one. + * + * Canonicalised before the reserved-name and duplicate checks, not after: the + * worker resolves `.` and `//` away when it materialises the bundle, so + * `./dbt_project.yml` and `dbt_project.yml` are two keys for one file on disk + * and either check is trivially walked past by the redundant spelling. */ +export function dbtModulePath( + path: string, + modules: Record | undefined +): { path: string } | { error: string } { + const canonical = canonicalModulePath(path) + if ('error' in canonical) return canonical + if (canonical.path === DBT_DESCRIPTOR) { + return { + error: `${DBT_DESCRIPTOR} is the descriptor, edited from the tree — it cannot also be a file` + } + } + if (!dbtModuleLang(canonical.path)) { + return { error: `File must end with one of: ${DBT_MODULE_EXTENSIONS.join(', ')}` } + } + const clash = findModulePathClash(modules, canonical.path) + if (clash) return { error: `${clash} already exists` } + return canonical +} + +/** Why this path cannot be a file, or `undefined` when it can. */ +export function dbtPathError( + path: string, + modules: Record | undefined +): string | undefined { + if (!path.trim()) return undefined + const resolved = dbtModulePath(path, modules) + return 'error' in resolved ? resolved.error : undefined +} + +/** A new file's starting content. A model compiles on its own, so it is runnable + * before it is edited; anything else starts empty rather than with a guess at + * which dbt schema it is. */ +export function dbtDefaultContent(filePath: string): string { + return filePath.endsWith('.sql') ? 'select 1 as id\n' : '' +} + +/** + * What `dbt build --select` should be given for an open file, or `undefined` + * when the file is not a model. + * + * Only files under the project's `model-paths` are models; a project also holds + * macros, analyses and singular tests, all `.sql`, none of them selectable by + * name. The selector is package-qualified because a bare leaf name also matches a + * dependency package's model of the same name. + */ +export function dbtModelSelector( + modules: Record, + filePath: string +): string | undefined { + // `.py` as well as `.sql`: a dbt Python model is a model, and leaving it out + // would run the whole project to check one file — the larger warehouse bill + // this narrowing exists to avoid. + const ext = ['.sql', '.py'].find((e) => filePath.endsWith(e)) + if (!ext) return undefined + const projectKey = dbtProjectFileKey(modules) + let project: any + try { + project = YAML.parse((projectKey ? modules[projectKey] : undefined)?.content ?? '') + } catch { + return undefined + } + if (!project?.name) return undefined + const modelPaths: string[] = Array.isArray(project['model-paths']) + ? project['model-paths'] + : ['models'] + if (!modelPaths.some((d) => filePath === d || filePath.startsWith(d + '/'))) return undefined + const name = filePath.split('/').pop()!.slice(0, -ext.length) + return `${name},package:${project.name}` +} + +/** The descriptor's `engine` and `profile.warehouse`, for the editor's header. + * Best effort: a descriptor mid-edit is often not valid YAML, and a header that + * blanked on every keystroke would be worse than one that lags. */ +export function dbtDescriptorSummary(content: string): { + engine: string + warehouse: string +} { + try { + const d = YAML.parse(content ?? '') + return { + engine: typeof d?.engine === 'string' ? d.engine : 'dbt-core-1x', + // A descriptor naming its own `profiles_yml` and no warehouse has no + // warehouse identity at all, which is what leaves it with no graph. + warehouse: + typeof d?.profile?.warehouse === 'string' + ? d.profile.warehouse + : d?.profile?.profiles_yml + ? 'own profiles.yml' + : 'main' + } + } catch { + return { engine: 'dbt-core-1x', warehouse: 'main' } + } +} diff --git a/frontend/src/lib/components/dbt/runStatus.svelte.ts b/frontend/src/lib/components/dbt/runStatus.svelte.ts new file mode 100644 index 0000000000..aaca172717 --- /dev/null +++ b/frontend/src/lib/components/dbt/runStatus.svelte.ts @@ -0,0 +1,100 @@ +/** + * What a dbt run is doing to each relation on a model graph, live and then + * settled. + * + * Two sources, in that order of authority: + * + * * the run's own `run_results.json`, once it has one — joined on dbt's + * `unique_id`, which is what both sides carry. The per-relation state table + * holds ONE row per relation stamped with its last writer, so reading that + * for a finished run would show it a later run's outcomes. + * * `dbt_run_progress` while it is in flight, polled. Only `dbt-core-1x` emits + * the node events behind it, and only a database-connected worker records + * them, so on the other engines and on agent workers this stays empty and the + * graph colours at the end instead. + * + * One definition, used by the run page and by the editor: a second one would be + * a second answer to "what colour is this model right now". + */ +import { JobService } from '$lib/gen' +import type { AssetGraphResponse, AssetRunState } from '$lib/components/assets/AssetGraph/types' +import { parseDbtRun, relationOutcome } from './parseDbtRun' + +export function useDbtRunStatus(opts: { + workspace: () => string | undefined + /** The run whose progress to show, or `undefined` for a graph with no run. */ + jobId: () => string | undefined + running: () => boolean + /** The finished job's result, which carries a status per dbt node. */ + result: () => unknown + /** The graph on screen, for the `unique_id` → relation mapping. */ + graph: () => AssetGraphResponse | undefined + /** Guards every response against a navigation: a poll that outlives the run + * it was issued for would colour the next one's models. */ + generation: () => number + destroyed: () => boolean +}) { + let polled = $state>(new Map()) + + async function load() { + const ws = opts.workspace() + const id = opts.jobId() + if (!ws || !id) return + const gen = opts.generation() + try { + const rows = await JobService.getRunProgress({ workspace: ws, id }) + if (gen !== opts.generation() || opts.destroyed()) return + const next = new Map() + for (const r of rows) { + next.set(`asset:${r.asset_kind}:${r.asset_path}`, { + status: r.status, + rowCount: r.row_count + }) + } + polled = next + } catch { + // A progress hiccup must not blank the graph. + } + } + + let run = $derived(parseDbtRun(opts.result())) + + let settled = $derived.by(() => { + if (opts.running()) return undefined + const g = opts.graph() + if (!run?.nodes?.length || !g) return undefined + const assetByNode = new Map() + for (const a of g.assets) { + if (a.dbt?.unique_id) assetByNode.set(a.dbt.unique_id, `asset:${a.kind}:${a.path}`) + } + const out = new Map() + for (const n of run.nodes) { + const id = assetByNode.get(n.unique_id) + const outcome = id && relationOutcome(n.status, n.outcome) + // A test or an analysis matches no relation, and a skipped node says + // nothing about one; both are left uncoloured rather than guessed at. + if (id && outcome) out.set(id, { status: outcome, rowCount: n.rows_affected }) + } + return out.size > 0 ? out : undefined + }) + + return { + /** Poll once. The caller owns the cadence — a run page ticks while its job + * runs, the editor while a build does. */ + load, + /** Drop what a previous run left, so its colours do not survive into the + * next one. */ + reset: () => (polled = new Map()), + get status() { + return settled ?? polled + }, + /** Whether the run settled itself, which is what tells a caller it no + * longer needs to poll. */ + get isSettled() { + return settled != undefined + }, + get run() { + return run + } + } +} diff --git a/frontend/src/lib/components/details/OnBehalfOfBadge.svelte b/frontend/src/lib/components/details/OnBehalfOfBadge.svelte new file mode 100644 index 0000000000..13bd94b122 --- /dev/null +++ b/frontend/src/lib/components/details/OnBehalfOfBadge.svelte @@ -0,0 +1,34 @@ + + +{#if onBehalfOf} + + {#snippet text()} + Every run of this {kind} is permissioned as {detailed}, whoever starts it. + {/snippet} + + On behalf of {onBehalfOf} + + +{/if} diff --git a/frontend/src/lib/components/flows/content/FlowInputs.svelte b/frontend/src/lib/components/flows/content/FlowInputs.svelte index 335a37b0c7..e275df3976 100644 --- a/frontend/src/lib/components/flows/content/FlowInputs.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputs.svelte @@ -12,7 +12,7 @@ import { Check, Code, Zap } from 'lucide-svelte' import SuspendDrawer from './SuspendDrawer.svelte' import { defaultScripts } from '$lib/stores' - import { defaultScriptLanguages, processLangs } from '$lib/scripts' + import { defaultScriptLanguages, processInlineLangs } from '$lib/scripts' import type { SupportedLanguage } from '$lib/common' import DefaultScripts from '$lib/components/DefaultScripts.svelte' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' @@ -48,7 +48,7 @@ let filter = $state('') let langs = $derived( - processLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) + processInlineLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) .map((l) => [defaultScriptLanguages[l], l]) .filter( (x) => $defaultScripts?.hidden == undefined || !$defaultScripts.hidden.includes(x[1]) diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index dc2cba93d4..7a55ca92fc 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -6,7 +6,7 @@ + + + + + + + + + + + diff --git a/frontend/src/lib/components/jobs/MissingWorkerTagAlert.svelte b/frontend/src/lib/components/jobs/MissingWorkerTagAlert.svelte new file mode 100644 index 0000000000..6d996e76e4 --- /dev/null +++ b/frontend/src/lib/components/jobs/MissingWorkerTagAlert.svelte @@ -0,0 +1,47 @@ + + +{#if served.current === false} +
+ + + {subject} run as Windmill jobs tagged {tag}, and no worker is currently listening to + that tag, so they stay queued until one is. If no worker group is meant to serve it, add + {tag} + to a group's worker tags on the workers page. + +
+{/if} diff --git a/frontend/src/lib/components/jobs/missingWorker.test.ts b/frontend/src/lib/components/jobs/missingWorker.test.ts new file mode 100644 index 0000000000..01f591aece --- /dev/null +++ b/frontend/src/lib/components/jobs/missingWorker.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +const getCompletedJobResultMaybe = vi.fn() +const getJob = vi.fn() +const cancelQueuedJob = vi.fn() +const existsWorkersWithTags = vi.fn() + +vi.mock('$lib/gen', () => ({ + JobService: { + getCompletedJobResultMaybe: (...a: unknown[]) => getCompletedJobResultMaybe(...(a as [])), + getJob: (...a: unknown[]) => getJob(...(a as [])), + cancelQueuedJob: (...a: unknown[]) => cancelQueuedJob(...(a as [])) + }, + WorkerService: { + existsWorkersWithTags: (...a: unknown[]) => existsWorkersWithTags(...(a as [])) + } +})) + +import { pollJobResult } from './utils' +import { hasWorkerForTag, NoWorkerForTagError } from './missingWorker' + +function settlementTracker(promise: Promise) { + const state = { settled: false } + promise.then( + () => (state.settled = true), + () => (state.settled = true) + ) + return state +} + +beforeEach(() => { + getCompletedJobResultMaybe.mockReset() + getJob.mockReset() + cancelQueuedJob.mockReset() + existsWorkersWithTags.mockReset() + getCompletedJobResultMaybe.mockResolvedValue({ completed: false }) + getJob.mockResolvedValue({ type: 'QueuedJob', running: false, tag: 'postgresql' }) + cancelQueuedJob.mockResolvedValue(undefined) + vi.useFakeTimers() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +// Long enough for the whole confirmation window (first probe + 2 intervals). +const PAST_CONFIRMATION_WINDOW_MS = 120_000 + +describe('pollJobResult', () => { + it('reports a queued read whose tag stays unserved without cancelling it', async () => { + existsWorkersWithTags.mockResolvedValue({ postgresql: false }) + + const promise = pollJobResult('job-1', 'ws') + const rejects = expect(promise).rejects.toBeInstanceOf(NoWorkerForTagError) + await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS) + await rejects + // The backlog is what the autoscaler scales up on: cancelling would stop a + // group coming back from zero from ever recovering. + expect(cancelQueuedJob).not.toHaveBeenCalled() + }) + + it('never abandons a write, and reports once why it is waiting', async () => { + existsWorkersWithTags.mockResolvedValue({ postgresql: false }) + const onNoWorkerForTag = vi.fn() + + const promise = pollJobResult('job-1', 'ws', { sideEffecting: true, onNoWorkerForTag }) + const tracker = settlementTracker(promise) + + await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS * 3) + // Reporting failure while the write stays executable would let it apply after + // the caller gave up and duplicate on retry; cancelling it first cannot be + // done atomically from the client. + expect(tracker.settled).toBe(false) + expect(cancelQueuedJob).not.toHaveBeenCalled() + expect(onNoWorkerForTag).toHaveBeenCalledTimes(1) + expect(onNoWorkerForTag).toHaveBeenCalledWith('postgresql') + + getCompletedJobResultMaybe.mockResolvedValue({ completed: true, success: true, result: 7 }) + await vi.advanceTimersByTimeAsync(3_000) + await expect(promise).resolves.toBe(7) + }) + + it('does not give up while a worker group could still be coming up', async () => { + // A worker group booting is absent from worker_ping exactly like an unserved + // tag; only a run of empty lookups distinguishes them. + existsWorkersWithTags.mockResolvedValueOnce({ postgresql: false }) + existsWorkersWithTags.mockResolvedValueOnce({ postgresql: false }) + existsWorkersWithTags.mockResolvedValue({ postgresql: true }) + + const promise = pollJobResult('job-1', 'ws') + const tracker = settlementTracker(promise) + + await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS) + expect(tracker.settled).toBe(false) + expect(cancelQueuedJob).not.toHaveBeenCalled() + }) + + it('keeps waiting on a job queued behind a busy worker that serves its tag', async () => { + existsWorkersWithTags.mockResolvedValue({ postgresql: true }) + + const promise = pollJobResult('job-1', 'ws') + const tracker = settlementTracker(promise) + + await vi.advanceTimersByTimeAsync(PAST_CONFIRMATION_WINDOW_MS) + expect(tracker.settled).toBe(false) + + getCompletedJobResultMaybe.mockResolvedValue({ completed: true, success: true, result: 42 }) + await vi.advanceTimersByTimeAsync(3_000) + await expect(promise).resolves.toBe(42) + }) +}) + +describe('hasWorkerForTag', () => { + it('treats an answer it did not get as a worker being there', async () => { + // `existsWorkersWithTags` returns an empty map when TAGS_ARE_SENSITIVE hides + // the tag from the caller. Reading that as "unserved" would diagnose a + // perfectly healthy instance. + existsWorkersWithTags.mockResolvedValue({}) + await expect(hasWorkerForTag('ws', 'postgresql')).resolves.toBe(true) + }) +}) diff --git a/frontend/src/lib/components/jobs/missingWorker.ts b/frontend/src/lib/components/jobs/missingWorker.ts new file mode 100644 index 0000000000..e392e06d3e --- /dev/null +++ b/frontend/src/lib/components/jobs/missingWorker.ts @@ -0,0 +1,76 @@ +import { JobService, WorkerService } from '$lib/gen' + +/** + * A queued job whose tag no running worker serves is never picked up: without + * this the UI polls until the server-side `run_wait_result` timeout (10min by + * default) or, for the client-side pollers, forever. + * + * The most common cause is a language that defaults to a native tag + * (`postgresql`, `mysql`, `bigquery`, …) on an instance whose worker groups + * only declare the default tags. + */ +export class NoWorkerForTagError extends Error { + tag: string + + constructor(tag: string) { + super( + `No worker has been listening to the tag "${tag}" while this job waited, so it was never ` + + `picked up. It stays queued and will run once a worker with that tag comes online. ` + + `Add "${tag}" to the worker tags of one of your worker groups (Workers page), or run a ` + + `worker that serves it.` + ) + this.name = 'NoWorkerForTagError' + this.tag = tag + } +} + +/** Shown while a write waits, which is never abandoned (see `sideEffecting`). */ +export function queuedWithoutWorkerMessage(tag: string): string { + return ( + `No worker is listening to the tag "${tag}", so this operation is queued and will only run ` + + `once one is. Add "${tag}" to the worker tags of one of your worker groups (Workers page), ` + + `or run a worker that serves it.` + ) +} + +/** How long a job may sit un-started before the first lookup for a worker serving its tag. */ +export const NO_WORKER_FIRST_PROBE_MS = 10_000 +/** How long to wait between lookups while the job stays queued. */ +export const NO_WORKER_PROBE_INTERVAL_MS = 40_000 +/** + * How many consecutive lookups must come back empty before the caller stops + * waiting. A worker group scaling from zero, or every worker down for a rollout, + * is indistinguishable from an unserved tag in any single lookup, so no single + * empty reading is acted on. + */ +export const NO_WORKER_CONFIRMATIONS = 3 + +/** + * Whether any worker pinged in the last minute declares `tag`. Unknown answers + * (the endpoint returns an empty map when tags are sensitive and the caller may + * not see them) count as "yes", so an opaque instance never gets a wrong + * diagnosis. + */ +export async function hasWorkerForTag(workspace: string, tag: string): Promise { + const existing = await WorkerService.existsWorkersWithTags({ workspace, tags: tag }) + return existing[tag] !== false +} + +/** + * The tag of `jobId` when it is still queued and no worker serves it, else + * undefined. Never throws: a failed lookup means "can't tell", and the caller + * keeps waiting rather than reporting a cause it did not establish. + */ +export async function missingWorkerTagOfQueuedJob( + workspace: string, + jobId: string +): Promise { + try { + const job = await JobService.getJob({ workspace, id: jobId, noCode: true, noLogs: true }) + if (job.type !== 'QueuedJob' || job.running || !job.tag) return undefined + return (await hasWorkerForTag(workspace, job.tag)) ? undefined : job.tag + } catch (err) { + console.warn('Could not determine whether a worker serves the job tag', err) + return undefined + } +} diff --git a/frontend/src/lib/components/jobs/utils.ts b/frontend/src/lib/components/jobs/utils.ts index 294df7c3ff..735484c1f2 100644 --- a/frontend/src/lib/components/jobs/utils.ts +++ b/frontend/src/lib/components/jobs/utils.ts @@ -1,4 +1,11 @@ import { JobService, type RunScriptByPathData, type RunScriptPreviewData } from '$lib/gen' +import { + missingWorkerTagOfQueuedJob, + NoWorkerForTagError, + NO_WORKER_CONFIRMATIONS, + NO_WORKER_FIRST_PROBE_MS, + NO_WORKER_PROBE_INTERVAL_MS +} from './missingWorker' function isRunScriptByPathData( arg: RunScriptPreviewData | RunScriptByPathData @@ -9,13 +16,26 @@ function isRunScriptByPathData( type RunScriptOptions = { maxRetries?: number withJobData?: boolean + /** Set to false to keep polling a job that no worker can pick up. */ + failIfNoWorkerForTag?: boolean + /** + * The job writes (insert/update/delete, DDL, arbitrary SQL). Such a job is + * never given up on: reporting it as failed while it stays executable invites + * a duplicate retry, and cancelling it first is not something the client can + * do atomically: a worker can claim it between the tag probe and the cancel, + * and a soft-cancelled statement may already have committed. `onNoWorkerForTag` + * is what tells the user why it is waiting. + */ + sideEffecting?: boolean + /** Called once when the job has stayed queued on a tag no worker serves. */ + onNoWorkerForTag?: (tag: string) => void } /** * @function runScript * @param {RunScriptPreviewData | RunScriptByPathData} data - Data for running the script. * @returns {Promise} A UUID representing the running script. - * + * * @example * const uuid = await runScript(data) */ @@ -29,6 +49,15 @@ export async function runScript(data: RunScriptPreviewData | RunScriptByPathData return uuid } +/** Tight at first so a quick job feels instant, then slower: a schema + * introspection or a DDL migration can run for minutes, and a fixed sub-second + * tick would cost hundreds of round-trips for it. */ +function pollDelayMs(poll: number): number { + if (poll < 4) return 375 + if (poll < 12) return 750 + return 2000 +} + /** * @function pollJobResult * @description Polls a job result by UUID until success, failure, or max retries reached. @@ -36,19 +65,35 @@ export async function runScript(data: RunScriptPreviewData | RunScriptByPathData * @param {string} workspace - Workspace identifier. * @param {RunScriptOptions} [options] - Optional settings like retries and job data inclusion. * @returns {Promise} Final job result or throws error if it fails. - * + * * @example * const result = await pollJobResult(uuid, 'my-workspace', { maxRetries: 5, withJobData: true }); */ export async function pollJobResult( uuid: string, workspace: string, - { maxRetries = 7, withJobData }: RunScriptOptions = {} + { + maxRetries = 7, + withJobData, + failIfNoWorkerForTag = true, + sideEffecting = false, + onNoWorkerForTag + }: RunScriptOptions = {} ): Promise { let attempts = 0 + let polls = 0 + // `attempts` only advances on errors, so a queued job would poll forever. The + // one case that never resolves on its own is a tag no worker serves, which + // takes NO_WORKER_CONFIRMATIONS consecutive empty lookups to establish, since + // a worker group booting reads like an unserved tag in any single one. + let noWorkerProbeAt = Date.now() + NO_WORKER_FIRST_PROBE_MS + let unservedProbes = 0 + let reportedNoWorker = false while (attempts < maxRetries) { try { - await new Promise((resolve) => setTimeout(resolve, 500 * (attempts || 0.75))) + await new Promise((resolve) => + setTimeout(resolve, attempts ? 500 * attempts : pollDelayMs(polls++)) + ) const job = await JobService.getCompletedJobResultMaybe({ id: uuid, workspace @@ -65,8 +110,26 @@ export async function pollJobResult( if (typeof errorMsg !== 'string') errorMsg = undefined console.error('JOB FAILED', job.result) throw new Error(errorMsg ?? 'Job failed') + } else if (failIfNoWorkerForTag && Date.now() >= noWorkerProbeAt) { + const tag = await missingWorkerTagOfQueuedJob(workspace, uuid) + noWorkerProbeAt = Date.now() + NO_WORKER_PROBE_INTERVAL_MS + unservedProbes = tag ? unservedProbes + 1 : 0 + if (tag && unservedProbes >= NO_WORKER_CONFIRMATIONS) { + if (!reportedNoWorker) { + reportedNoWorker = true + onNoWorkerForTag?.(tag) + } + // Reads give up the wait but leave the job queued: cancelling one would + // remove the very backlog the autoscaler scales up on, so a group coming + // back from zero (300s cooldown) would never recover. Writes keep + // waiting instead (see `sideEffecting`). + if (!sideEffecting) throw new NoWorkerForTagError(tag) + } } } catch (e) { + if (e instanceof NoWorkerForTagError) { + throw e + } if (attempts == maxRetries) { throw e } diff --git a/frontend/src/lib/components/jobs/writingJob.ts b/frontend/src/lib/components/jobs/writingJob.ts new file mode 100644 index 0000000000..d1c32ba826 --- /dev/null +++ b/frontend/src/lib/components/jobs/writingJob.ts @@ -0,0 +1,12 @@ +import { sendUserToast } from '$lib/toast' +import { queuedWithoutWorkerMessage } from './missingWorker' + +/** + * Poll options for a job that writes (row edits, DDL, arbitrary SQL). Such a job + * is never abandoned (see `sideEffecting` in `pollJobResult`), so this is what + * explains the wait when it sits on a tag no worker serves. + */ +export const writingJobOptions = { + sideEffecting: true, + onNoWorkerForTag: (tag: string) => sendUserToast(queuedWithoutWorkerMessage(tag), true) +} diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 31e99dbe89..7dc2b99d7e 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -54,6 +54,7 @@ } from 'lucide-svelte' import DraggableTabs, { type TabItem } from '$lib/components/common/tabs/DraggableTabs.svelte' import { runScriptAndPollResult } from '../jobs/utils' + import { writingJobOptions } from '../jobs/writingJob' import { RawAppHistoryManager } from './RawAppHistoryManager.svelte' import { sendUserToast } from '$lib/utils' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' @@ -1020,14 +1021,17 @@ } try { - const result = await runScriptAndPollResult({ - workspace: opWorkspace, - requestBody: { - language: 'postgresql', - content: sql, - args: { database: `datatable://${datatableName}` } - } - }) + const result = await runScriptAndPollResult( + { + workspace: opWorkspace, + requestBody: { + language: 'postgresql', + content: sql, + args: { database: `datatable://${datatableName}` } + } + }, + writingJobOptions + ) // If newTable was specified and the query succeeded, add it to data.tables if (newTable) { diff --git a/frontend/src/lib/components/raw_apps/RawAppYamlEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppYamlEditor.svelte index 59b0885989..4ac143976b 100644 --- a/frontend/src/lib/components/raw_apps/RawAppYamlEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppYamlEditor.svelte @@ -25,14 +25,7 @@ onApply: (update: RawAppYamlUpdate) => void } - let { - drawer = $bindable(), - summary, - files, - runnables, - data, - onApply - }: Props = $props() + let { drawer = $bindable(), summary, files, runnables, data, onApply }: Props = $props() let code = $state('') let initialCode = $state('') @@ -106,6 +99,7 @@ minHeight={editorHeight} bind:code lang="yaml" + leadingChangeSync />
{/await} diff --git a/frontend/src/lib/components/runs/JobDetailFieldConfig.ts b/frontend/src/lib/components/runs/JobDetailFieldConfig.ts index c0148cfc24..c79ea6a0a9 100644 --- a/frontend/src/lib/components/runs/JobDetailFieldConfig.ts +++ b/frontend/src/lib/components/runs/JobDetailFieldConfig.ts @@ -1,5 +1,5 @@ import type { Job } from '$lib/gen' -import { triggerIconMap } from '$lib/components/triggers/utils' +import { triggerDisplayNamesMap, triggerIconMap } from '$lib/components/triggers/utils' import { formatMemory, jobDisplayDurationMs } from '$lib/utils' import { flowPathToHref } from '$lib/scripts' import { Calendar, Bot } from 'lucide-svelte' @@ -125,14 +125,13 @@ export function getTriggerInfo(job: Job): { type: string; icon: any; detail?: st } // Check for trigger type from job trigger_kind if available - if ('trigger_kind' in job) { - const triggerKind = (job as any).trigger_kind - if (triggerKind && triggerIconMap[triggerKind]) { - return { - type: getTriggerDisplayName(triggerKind), - icon: triggerIconMap[triggerKind], - detail: triggerKind - } + const triggerKind = job.trigger_kind + if (triggerKind && triggerKind in triggerIconMap) { + // No detail: unlike the schedule branch above, the kind carries no path to add, + // and repeating it would render as "Webhook: webhook". + return { + type: triggerDisplayNamesMap[triggerKind], + icon: triggerIconMap[triggerKind] } } @@ -154,28 +153,6 @@ export function getTriggerInfo(job: Job): { type: string; icon: any; detail?: st return null } -/** - * Gets human-readable display name for trigger kinds - */ -function getTriggerDisplayName(triggerKind: string): string { - const displayNames: Record = { - webhook: 'Webhook', - http: 'HTTP', - websocket: 'WebSocket', - postgres: 'PostgreSQL', - kafka: 'Kafka', - nats: 'NATS', - mqtt: 'MQTT', - sqs: 'SQS', - gcp: 'GCP Pub/Sub', - email: 'Email', - schedule: 'Schedule', - app: 'App', - ui: 'UI' - } - return displayNames[triggerKind] || triggerKind.toUpperCase() -} - /** * Field configurations for all possible fields */ @@ -556,6 +533,12 @@ export function getRelevantFields(job: Job): FieldConfig[] { // Always show schedule_path when it exists, regardless of category configuration return job.schedule_path !== null && job.schedule_path !== undefined } + if (fieldName === 'trigger_info' && job.trigger_kind) { + // A stamped trigger kind is how the run was started, whatever the category — but + // only one `getTriggerInfo` can name, or the row renders empty. A schedule is + // left out because its own field already says the same thing. + return !job.schedule_path && job.trigger_kind in triggerIconMap + } return fieldsPresence[fieldName] }) .map((fieldName) => fieldConfigs[fieldName]) diff --git a/frontend/src/lib/components/scriptModulePath.test.ts b/frontend/src/lib/components/scriptModulePath.test.ts new file mode 100644 index 0000000000..d90624d10d --- /dev/null +++ b/frontend/src/lib/components/scriptModulePath.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect } from 'vitest' +import { canonicalModulePath, findModulePathClash } from './scriptModulePath' + +describe('canonicalModulePath', () => { + // The pair the duplicate and reserved-name checks exist to catch: both + // spellings resolve to the same file in the job directory. + it('rewrites a redundant spelling to the file it names', () => { + expect(canonicalModulePath('./dbt_project.yml')).toEqual({ path: 'dbt_project.yml' }) + expect(canonicalModulePath('models//x.sql')).toEqual({ path: 'models/x.sql' }) + expect(canonicalModulePath(' ./models/./sub//x.sql ')).toEqual({ + path: 'models/sub/x.sql' + }) + expect(canonicalModulePath('models/x.sql')).toEqual({ path: 'models/x.sql' }) + }) + + it('refuses a path that leaves the bundle', () => { + expect(canonicalModulePath('../x.sql')).toHaveProperty('error') + expect(canonicalModulePath('models/../../x.sql')).toHaveProperty('error') + expect(canonicalModulePath('/etc/x.sql')).toHaveProperty('error') + expect(canonicalModulePath('./')).toHaveProperty('error') + }) + + // Matches the worker's own rule: `..` is traversal only as a whole segment. + it('takes dots inside a name as part of the name', () => { + expect(canonicalModulePath('models/weird..name.sql')).toEqual({ + path: 'models/weird..name.sql' + }) + }) +}) + +describe('findModulePathClash', () => { + // A bundle pushed by the CLI can hold a non-canonical key, so the clash has to + // be found from either side, and named the way the tree shows it. + it('finds a key that resolves to the same file, however either is spelled', () => { + const modules = { './dbt_project.yml': {}, 'models/x.sql': {} } + expect(findModulePathClash(modules, 'dbt_project.yml')).toBe('./dbt_project.yml') + expect(findModulePathClash(modules, 'models/x.sql')).toBe('models/x.sql') + expect(findModulePathClash(modules, 'models/y.sql')).toBeUndefined() + expect(findModulePathClash(undefined, 'models/x.sql')).toBeUndefined() + }) + + // The worker does not trim path components, so an imported `x.sql ` is its + // own file and must not stand in the way of adding `x.sql`. + it('does not fold a key whose name carries whitespace', () => { + expect(findModulePathClash({ 'models/x.sql ': {} }, 'models/x.sql')).toBeUndefined() + }) + + // A rename must not stop at the module being renamed: with both spellings in + // the bundle, that would hide the other one and overwrite its content. + it('keeps looking past the key being renamed', () => { + const modules = { './models/x.sql': {}, 'models/x.sql': {} } + expect(findModulePathClash(modules, 'models/x.sql', './models/x.sql')).toBe('models/x.sql') + expect( + findModulePathClash({ './models/x.sql': {} }, 'models/x.sql', './models/x.sql') + ).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/scriptModulePath.ts b/frontend/src/lib/components/scriptModulePath.ts new file mode 100644 index 0000000000..63271135c6 --- /dev/null +++ b/frontend/src/lib/components/scriptModulePath.ts @@ -0,0 +1,64 @@ +function canonicalize(path: string): { path: string } | { error: string } { + if (path.startsWith('/')) { + return { error: `File path must be relative, without a leading /` } + } + const segments = path.split('/').filter((s) => s !== '' && s !== '.') + if (segments.includes('..')) { + return { error: `File path cannot contain ..` } + } + if (segments.length === 0) { + return { error: `File name cannot be empty` } + } + return { path: segments.join('/') } +} + +/** + * The canonical spelling of a script module's path (the key of the module + * bundle), or the reason it cannot be one. + * + * The worker resolves `.` and `//` away when it materialises the bundle into + * the job directory, so `./dbt_project.yml` and `dbt_project.yml` are two keys + * for one file on disk; the bundle is a Rust `HashMap`, so which content lands + * there is undefined. Canonicalising before the duplicate and reserved-name + * checks is what keeps them from being walked past. + * + * Redundant spellings are rewritten rather than refused: they name the file the + * user meant, and the tree shows the canonical form regardless. `..` and + * absolute paths name a file OUTSIDE the bundle, which has no canonical form + * inside it, so they are refused here — the worker drops them, which would + * otherwise show up as a file that was added and then silently never written. + * + * Surrounding whitespace is dropped because this is what someone typed into a + * text box. A key already in a bundle gets no such courtesy — see + * `findModulePathClash`. + */ +export function canonicalModulePath(path: string): { path: string } | { error: string } { + return canonicalize(path.trim()) +} + +/** + * The existing module key that would land on the same file as `canonicalPath`, + * spelled as the bundle holds it (which is what the file tree shows). + * + * Keys already in the bundle are not canonical either: nothing on the push path + * rewrites them, so a project imported with `./dbt_project.yml` in it must still + * refuse a second `dbt_project.yml`. They are matched WITHOUT trimming, because + * the worker does not trim path components: an imported `x.sql ` is its own file + * on disk and must not stand in the way of adding `x.sql`. + * + * `ignoreKey` is the module being renamed. It has to be skipped inside the + * search rather than compared against the result: a bundle can hold BOTH + * spellings, and stopping at the renamed one would hide the other and let the + * rename overwrite it. + */ +export function findModulePathClash( + modules: Record | null | undefined, + canonicalPath: string, + ignoreKey?: string +): string | undefined { + return Object.keys(modules ?? {}).find((key) => { + if (key === ignoreKey) return false + const canonical = canonicalize(key) + return 'path' in canonical && canonical.path === canonicalPath + }) +} diff --git a/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte b/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte index 28992cf72c..5c01c84918 100644 --- a/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte +++ b/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte @@ -289,7 +289,7 @@ {/if} {#if selected === 'OpenAPI' || (selected === 'OpenAPI_File' && !emptyStringTrimmed(openApiFile)) || (selected === 'OpenAPI_URL' && !emptyStringTrimmed(openApiUrl))} {#key forceRerender} - + {/key} {/if}
{:else} + {#if loadError}
Could not read applied status from the data table: {loadError} diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 069ef6093d..254a20fe80 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -87,6 +87,7 @@ import { clone } from '$lib/utils' import SettingsFooter from './SettingsFooter.svelte' import Alert from '../common/alert/Alert.svelte' + import MissingWorkerTagAlert from '../jobs/MissingWorkerTagAlert.svelte' import { isCloudHosted } from '$lib/cloud' type Props = { @@ -252,6 +253,8 @@ {/if} + + diff --git a/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte b/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte new file mode 100644 index 0000000000..d472ce6d08 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/DbtSettings.svelte @@ -0,0 +1,173 @@ + + + + + + Where dbt projects in this workspace run. A project names a warehouse by name in its descriptor (profile.warehouse) and reaches + {DEFAULT_WAREHOUSE} when it names none, so a project carries no + connection of its own. The name is also what its tables are keyed on in the asset graph (dbt://{DEFAULT_WAREHOUSE}/schema/table), so two projects on one warehouse share their nodes. Each entry points at a resource, and + configuring one here is what makes it available: anyone who may run a dbt script builds with it + and reads its models, without being granted the resource, the same bargain workspace object + storage makes. + + + + + + Name + Resource + Target + + + + + {#each dbtSettings.warehouses as warehouse, i (i)} + + + + + + + + + + + + + + + + + + onDiscard?.()} + saveLabel="Save dbt warehouses" +/> diff --git a/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte b/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte index cbf12c6314..dcbef04a51 100644 --- a/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte @@ -25,6 +25,7 @@ settings: boolean key: boolean workspaceDependencies: boolean + dataTableMigrations: boolean } let { @@ -35,7 +36,8 @@ 'flow', 'app', 'folder', - 'workspacedependencies' + 'workspacedependencies', + 'datatablemigration' ] as GitSyncObjectType[]), exclude_types_override = $bindable([] as GitSyncObjectType[]), isLegacyRepo = false, @@ -76,7 +78,8 @@ triggers: effectiveIncludeTypes.includes('trigger'), settings: effectiveIncludeTypes.includes('settings'), key: effectiveIncludeTypes.includes('key'), - workspaceDependencies: effectiveIncludeTypes.includes('workspacedependencies') + workspaceDependencies: effectiveIncludeTypes.includes('workspacedependencies'), + dataTableMigrations: effectiveIncludeTypes.includes('datatablemigration') }) // Tab selection for filter kinds @@ -99,7 +102,8 @@ triggers: 'trigger', settings: 'settings', key: 'key', - workspaceDependencies: 'workspacedependencies' + workspaceDependencies: 'workspacedependencies', + dataTableMigrations: 'datatablemigration' } if (value) { @@ -321,6 +325,14 @@ options={{ right: 'Workspace dependencies' }} />
+
+ updateIncludeType('dataTableMigrations', e.detail)} + options={{ right: 'Data table migrations' }} + /> +
diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.ts index 387dc82ade..abc48a3ad6 100644 --- a/frontend/src/lib/components/workspaceSettings/projectInstall.ts +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.ts @@ -23,6 +23,7 @@ import { updateRawAppPolicy } from '$lib/sharedUtils' import { apiErrorMessage as errorMessage } from '$lib/utils' import type { App } from '$lib/components/apps/types' import { runScriptAndPollResult } from '$lib/components/jobs/utils' +import { writingJobOptions } from '$lib/components/jobs/writingJob' import { classifyPath, collectExportVarPaths, @@ -233,14 +234,17 @@ async function applyOneMigration( only: created.timestamp }) } else { - await runScriptAndPollResult({ - workspace, - requestBody: { - language: 'postgresql', - content: m.sql, - args: { database: `datatable://${m.datatable_name}` } - } - }) + await runScriptAndPollResult( + { + workspace, + requestBody: { + language: 'postgresql', + content: m.sql, + args: { database: `datatable://${m.datatable_name}` } + } + }, + writingJobOptions + ) } } diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 1e0a36af50..a23bb4af0a 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -44,6 +44,14 @@ import initYamlParser, { parse_ansible, parse_ansible_delegate } from 'windmill-parser-wasm-yaml' +// `parse_dbt` is newer than the published `windmill-parser-wasm-yaml`, and a +// NAMED import of an export a package does not have is a link-time failure — +// which takes this whole module down, and with it every language's inference. +// Reached through the namespace instead, so a package predating it degrades to +// "no schema inferred for dbt" (the deployed script's own, derived server-side +// by `dbt_arg_schema`, is unaffected). +import * as yamlParser from 'windmill-parser-wasm-yaml' +const parse_dbt: ((code: string) => string) | undefined = (yamlParser as any).parse_dbt import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp' import initNuParser, { parse_nu } from 'windmill-parser-wasm-nu' import initJavaParser, { parse_java } from 'windmill-parser-wasm-java' @@ -521,6 +529,12 @@ export async function inferArgs( } catch { inferedSchema = parseRSignatureFallback(code) } + } else if (language == 'dbt') { + // Absent on a parser package predating dbt: the editor keeps whatever + // schema it has rather than clearing the run form to nothing. + if (!parse_dbt) return null + await initWasmYaml() + inferedSchema = JSON.parse(parse_dbt(code)) // for related places search: ADD_NEW_LANG } else { return null diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 70dabb01f6..f67c0415f0 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -709,7 +709,7 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "language": { "type": "string", - "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative" + "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" }, "kind": { "type": "string", @@ -1218,7 +1218,7 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "language": { "type": "string", - "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative" + "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" }, "tag": { "type": "string" @@ -1250,7 +1250,7 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "language": { "type": "string", - "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative" + "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" }, "lock": { "type": "string", diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 91e310f5c9..178f4474e7 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -1345,6 +1345,43 @@ main <- function( return(toJSON(result, auto_unbox = TRUE)) } ` + +// A dbt script is a whole dbt project: the descriptor below is the script's +// content, and the project's own files live in its module bundle (the +// `