From 6564f5a814eaa40d2130bccc53d1f6ea7c0def86 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 4 Sep 2026 18:20:08 +0200 Subject: [PATCH] docs: qualify the any-language claim, and pin the dbt-script refusal `AssetKind::Dbt`'s contract (both enums), the two runtime guides and the deploy comment said a script of any language may declare a `dbt://` write, which the dbt-script refusal added last round contradicts. They now say "any language but dbt's own", with the reason: a project's writes are read from its manifest. The deploy-contract integration test covers that refusal for both annotations. Co-Authored-By: Claude Opus 5 (1M context) --- .../windmill-parser/src/asset_parser.rs | 4 +-- .../tests/dbt_materialize_target.rs | 26 +++++++++++++++++++ backend/windmill-api-scripts/src/scripts.rs | 7 ++--- backend/windmill-types/src/assets.rs | 7 ++--- backend/windmill-worker/src/worker.rs | 3 ++- docs/dbt-runtime.md | 12 +++++---- docs/ducklake-materialization.md | 4 +-- 7 files changed, 47 insertions(+), 16 deletions(-) diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 27583726fd..cebc31484d 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -31,8 +31,8 @@ pub enum AssetKind { Volume, /// A warehouse relation, `dbt:////`, the warehouse /// named as the workspace configures it. The scheme names the namespace dbt - /// made — a script of any language can declare a write to one — and the path - /// stays the relation. See `windmill_types::AssetKind::Dbt`. + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs index b8ca7405ce..42babe72ca 100644 --- a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -131,5 +131,31 @@ async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyh assert_eq!(resp.status(), 400); assert!(resp.text().await?.contains("u/test-user/project")); + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + Ok(()) } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 10ee8c0b8e..af17e18174 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1573,9 +1573,10 @@ async fn create_script_internal<'c>( // A non-DuckDB script never reaches that executor. // • `dbt:////` — a warehouse relation. Nothing // generates warehouse DDL, so the declaration is track-only (`manual`) - // and any language may make it: the script writes the relation, the - // worker records the materialization, and the relation's asset node is - // shared with whatever dbt model reads it. + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. // Any other kind would deploy, register a producer in the asset graph, then // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). // The managed-only checks (single trailing SELECT, no SQL args) come after — diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index d5b5354755..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -18,9 +18,10 @@ pub enum AssetKind { /// `` is the name the workspace configures it under. /// /// The SCHEME names the namespace dbt made rather than an exclusive - /// producer: dbt is what derives these relations from a project, and a - /// script of any language can DECLARE one it writes - /// (`// materialize manual dbt://…`). The PATH stays the physical relation, + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, /// because that is what two producers agree on: a mart one builds is a /// `source` the next reads, and their dbt `unique_id`s differ /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ded2cd2a28..8dc52ef936 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -5564,7 +5564,8 @@ async fn handle_code_execution_job( /// Nothing generates warehouse DDL, so the script issues its own write and this /// is the only thing that turns it into a `materialized_partition` row — the /// relation's last writer on the run page and the graph. Language-agnostic on -/// purpose: the DuckLake write engine is DuckDB's, this declaration is anyone's. +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. /// /// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` /// rows, not from this one, so a lost row costs the relation its last writer and diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index e98d46d49f..a86dbf8930 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -47,7 +47,7 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | -| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -163,9 +163,10 @@ put warehouse relations in the asset graph and is what derives them from a project; no other language *infers* one, and calling the kind something generic promised a parity with native Snowflake and BigQuery scripts that does not exist. A script can nonetheless DECLARE that it writes one — `// materialize manual -dbt:////`, in any language — and that declaration lands -on the same node the dbt model reading the relation does, because identity is the -relation rather than the tool. See "No cascade *from* dbt" below. +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -690,7 +691,8 @@ mode: nothing generates warehouse DDL, so the script issues its own write and Windmill records the outcome — the same `materialized_partition` row a DuckLake target lands, so the relation carries a last writer on the run page and the graph. It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration -is anyone's), and the recording happens in the generic job path +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path (`record_declared_warehouse_write`) rather than in an executor, for the same reason. Identity is unchanged — the physical relation — so the ingestion script and the dbt model reading it are one node, and a `source` declared on the relation diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index aa4a520196..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -608,8 +608,8 @@ examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion - **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; - a `dbt://` warehouse-relation target is the one any language may declare, and - it is track-only — see `docs/dbt-runtime.md`), and the SDK + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** —